def __repr__(self):
extra = "".join([", %s=%r" % (k, v)
for k, v in iteritems(self.__kwargs)])
if self.database is None:
return "DBRef(%r, %r%s)" % (self.collection, self.id, extra)
return "DBRef(%r, %r, %r%s)" % (self.collection, self.id,
self.database, extra)
def _deepcopy(self, x, memo=None):
"""Deepcopy helper for the data dictionary or list.
Regular expressions cannot be deep copied but as they are immutable we
don't have to copy them when cloning.
"""
if not hasattr(x, 'items'):
y, is_list, iterator = [], True, enumerate(x)
else:
y, is_list, iterator = {}, False, iteritems(x)
if memo is None:
memo = {}
val_id = id(x)
if val_id in memo:
return memo.get(val_id)
memo[val_id] = y
for key, value in iterator:
if isinstance(value, (dict, list)) and not isinstance(value, SON):
value = self._deepcopy(value, memo)
elif not isinstance(value, RE_TYPE):
value = copy.deepcopy(value, memo)
if is_list:
y.append(value)
else:
if not isinstance(key, RE_TYPE):
key = copy.deepcopy(key, memo)
y[key] = value
return y
def get_validated_options(options, warn=True):
"""Validate each entry in options and raise a warning if it is not valid.
Returns a copy of options with invalid entries removed.
:Parameters:
- `opts`: A dict of MongoDB URI options.
- `warn` (optional): If ``True`` then warnings will be logged and
invalid options will be ignored. Otherwise, invalid options will
cause errors.
"""
validated_options = {}
for opt, value in iteritems(options):
lower = opt.lower()
try:
validator = URI_OPTIONS_VALIDATOR_MAP.get(
lower, raise_config_error)
value = validator(opt, value)
except (ValueError, TypeError, ConfigurationError) as exc:
if warn:
warnings.warn(str(exc))
else:
raise
else:
validated_options[lower] = value
return validated_options
def _index_document(index_list):
"""Helper to generate an index specifying document.
Takes a list of (key, direction) pairs.
"""
if isinstance(index_list, collections.Mapping):
raise TypeError("passing a dict to sort/create_index/hint is not "
"allowed - use a list of tuples instead. did you "
"mean %r?" % list(iteritems(index_list)))
elif not isinstance(index_list, (list, tuple)):
raise TypeError("must use a list of (key, direction) pairs, "
"not: " + repr(index_list))
if not len(index_list):
raise ValueError("key_or_list must not be the empty list")
index = SON()
for (key, value) in index_list:
if not isinstance(key, string_type):
raise TypeError("first item in each key pair must be a string")
if not isinstance(value, (string_type, int, collections.Mapping)):
raise TypeError("second item in each key pair must be 1, -1, "
"'2d', 'geoHaystack', or another valid MongoDB "
"index specifier.")
index[key] = value
return index
def _encode_mapping(name, value, check_keys, opts):
"""Encode a mapping type."""
if _raw_document_class(value):
return b'\x03' + name + value.raw
data = b"".join([_element_to_bson(key, val, check_keys, opts)
for key, val in iteritems(value)])
return b"\x03" + name + _PACK_INT(len(data) + 5) + data + b"\x00"
def run_scenario(self):
# Set up
self.setUpCluster(scenario_def)
try:
with get_change_stream(
self.client, scenario_def, test
) as change_stream:
for operation in test["operations"]:
# Run specified operations
run_operation(self.client, operation)
num_expected_changes = len(test["result"]["success"])
changes = [
change_stream.next() for _ in range(num_expected_changes)
]
except OperationFailure as exc:
if test["result"].get("error") is None:
raise
expected_code = test["result"]["error"]["code"]
self.assertEqual(exc.code, expected_code)
else:
# Check for expected output from change streams
for change, expected_changes in zip(changes, test["result"]["success"]):
assert_dict_is_subset(change, expected_changes)
self.assertEqual(len(changes), len(test["result"]["success"]))
finally:
# Check for expected events
results = self.listener.results
for expectation in test["expectations"]:
for idx, (event_type, event_desc) in enumerate(iteritems(expectation)):
results_key = event_type.split("_")[1]
event = results[results_key][idx] if len(results[results_key]) > idx else None
check_event(event, event_desc)
def run_operation(self, collection, test):
# Iterate over all operations.
for opdef in test['operations']:
# Convert command from CamelCase to pymongo.collection method.
operation = camel_to_snake(opdef['name'])
# Get command handle on target entity (collection/database).
target_object = opdef.get('object', 'collection')
if target_object == 'database':
cmd = getattr(collection.database, operation)
elif target_object == 'collection':
collection = collection.with_options(**dict(
parse_collection_options(opdef.get(
'collectionOptions', {}))))
cmd = getattr(collection, operation)
else:
self.fail("Unknown object name %s" % (target_object,))
# Convert arguments to snake_case and handle special cases.
arguments = opdef['arguments']
options = arguments.pop("options", {})
for option_name in options:
arguments[camel_to_snake(option_name)] = options[option_name]
if operation == "bulk_write":
# Parse each request into a bulk write model.
requests = []
for request in arguments["requests"]:
bulk_model = camel_to_upper_camel(request["name"])
bulk_class = getattr(operations, bulk_model)
bulk_arguments = camel_to_snake_args(request["arguments"])
requests.append(bulk_class(**bulk_arguments))
arguments["requests"] = requests
else:
for arg_name in list(arguments):
c2s = camel_to_snake(arg_name)
# PyMongo accepts sort as list of tuples.
if arg_name == "sort":
sort_dict = arguments[arg_name]
arguments[arg_name] = list(iteritems(sort_dict))
# Named "key" instead not fieldName.
if arg_name == "fieldName":
arguments["key"] = arguments.pop(arg_name)
# Aggregate uses "batchSize", while find uses batch_size.
elif arg_name == "batchSize" and operation == "aggregate":
continue
# Requires boolean returnDocument.
elif arg_name == "returnDocument":
arguments[c2s] = arguments[arg_name] == "After"
else:
arguments[c2s] = arguments.pop(arg_name)
if opdef.get('error') is True:
with self.assertRaises(PyMongoError):
cmd(**arguments)
else:
result = cmd(**arguments)
self.check_result(opdef.get('result'), result)
def validate_collection(self, name_or_collection,
scandata=False, full=False):
"""Validate a collection.
Returns a dict of validation info. Raises CollectionInvalid if
validation fails.
With MongoDB < 1.9 the result dict will include a `result` key
with a string value that represents the validation results. With
MongoDB >= 1.9 the `result` key no longer exists and the results
are split into individual fields in the result dict.
:Parameters:
- `name_or_collection`: A Collection object or the name of a
collection to validate.
- `scandata`: Do extra checks beyond checking the overall
structure of the collection.
- `full`: Have the server do a more thorough scan of the
collection. Use with `scandata` for a thorough scan
of the structure of the collection and the individual
documents. Ignored in MongoDB versions before 1.9.
"""
name = name_or_collection
if isinstance(name, Collection):
name = name.name
if not isinstance(name, string_type):
raise TypeError("name_or_collection must be an instance of "
"%s or Collection" % (string_type.__name__,))
result = self.command("validate", _unicode(name),
scandata=scandata, full=full)
valid = True
# Pre 1.9 results
if "result" in result:
info = result["result"]
if info.find("exception") != -1 or info.find("corrupt") != -1:
raise CollectionInvalid("%s invalid: %s" % (name, info))
# Sharded results
elif "raw" in result:
for _, res in iteritems(result["raw"]):
if "result" in res:
info = res["result"]
if (info.find("exception") != -1 or
info.find("corrupt") != -1):
raise CollectionInvalid("%s invalid: "
"%s" % (name, info))
elif not res.get("valid", False):
valid = False
break
# Post 1.9 non-sharded results.
elif not result.get("valid", False):
valid = False
if not valid:
raise CollectionInvalid("%s invalid: %r" % (name, result))
return result
def transform_value(value):
if isinstance(value, list):
return [transform_value(v) for v in value]
elif isinstance(value, abc.Mapping):
return dict([
(k, transform_value(v))
for k, v in iteritems(value)])
else:
return value
def validate_options(opts):
"""Validates and normalizes options passed in a MongoDB URI.
Returns a new dictionary of validated and normalized options.
:Parameters:
- `opts`: A dict of MongoDB URI options.
"""
return dict([_validate(opt, val) for opt, val in iteritems(opts)])
def _normalize_options(options):
"""Renames keys in the options dictionary to their internally-used
names."""
normalized_options = {}
for key, value in iteritems(options):
optname = str(key).lower()
intname = INTERNAL_URI_OPTION_NAME_MAP.get(optname, key)
normalized_options[intname] = options[key]
return normalized_options
def check_event(event, expectation_dict):
if event is None:
raise AssertionError
for key, value in iteritems(expectation_dict):
if isinstance(value, dict):
assert_dict_is_subset(
getattr(event, key), value
)
else:
assert getattr(event, key) == value
def _json_convert(obj):
"""Recursive helper method that converts BSON types so they can be
converted into json.
"""
if hasattr(obj, 'iteritems') or hasattr(obj, 'items'): # PY3 support
return SON(((k, _json_convert(v)) for k, v in iteritems(obj)))
elif hasattr(obj, '__iter__') and not isinstance(obj, (text_type, bytes)):
return list((_json_convert(v) for v in obj))
try:
return default(obj)
except TypeError:
return obj
def assert_dict_is_subset(superdict, subdict):
"""Check that subdict is a subset of superdict."""
exempt_fields = ["documentKey", "_id"]
for key, value in iteritems(subdict):
if key not in superdict:
assert False
if isinstance(value, dict):
assert_dict_is_subset(superdict[key], value)
continue
if key in exempt_fields:
superdict[key] = "42"
assert superdict[key] == value
def _clone(self, deepcopy=True):
"""Internal clone helper."""
clone = self._clone_base()
values_to_clone = ("spec", "projection", "skip", "limit",
"max_time_ms", "max_await_time_ms", "comment",
"max", "min", "ordering", "explain", "hint",
"batch_size", "max_scan", "manipulate",
"query_flags", "modifiers", "collation")
data = dict((k, v) for k, v in iteritems(self.__dict__)
if k.startswith('_Cursor__') and k[9:] in values_to_clone)
if deepcopy:
data = self._deepcopy(data)
clone.__dict__.update(data)
return clone
def get_validated_options(options):
"""Validate each entry in options and raise a warning if it is not valid.
Returns a copy of options with invalid entries removed
"""
validated_options = {}
for opt, value in iteritems(options):
lower = opt.lower()
try:
validator = VALIDATORS.get(lower, raise_config_error)
value = validator(opt, value)
except (ValueError, ConfigurationError) as exc:
warnings.warn(str(exc))
else:
validated_options[lower] = value
return validated_options
def __init__(self, username, password, database, options):
options = dict([validate(opt, val) for opt, val in iteritems(options)])
self.__codec_options = _parse_codec_options(options)
self.__credentials = _parse_credentials(
username, password, database, options)
self.__local_threshold_ms = options.get('localthresholdms', 15)
# self.__server_selection_timeout is in seconds. Must use full name for
# common.SERVER_SELECTION_TIMEOUT because it is set directly by tests.
self.__server_selection_timeout = options.get(
'serverselectiontimeoutms', common.SERVER_SELECTION_TIMEOUT)
self.__pool_options = _parse_pool_options(options)
self.__read_preference = _parse_read_preference(options)
self.__replica_set_name = options.get('replicaset')
self.__write_concern = _parse_write_concern(options)
def _dict_to_bson(doc, check_keys, opts, top_level=True):
"""Encode a document to BSON."""
try:
elements = []
if top_level and "_id" in doc:
elements.append(_name_value_to_bson(b"_id\x00", doc["_id"],
check_keys, opts))
for (key, value) in iteritems(doc):
if not top_level or key != "_id":
elements.append(_element_to_bson(key, value,
check_keys, opts))
except AttributeError:
raise TypeError("encoder expected a mapping type but got: %r" % (doc,))
encoded = b"".join(elements)
return _PACK_INT(len(encoded) + 5) + encoded + b"\x00"
def run_scenario(self):
# Load data.
assert scenario_def['data'], "tests must have non-empty data"
self.db.test.drop()
self.db.test.insert_many(scenario_def['data'])
# Convert command from CamelCase to pymongo.collection method.
operation = camel_to_snake(test['operation']['name'])
cmd = getattr(self.db.test, operation)
# Convert arguments to snake_case and handle special cases.
arguments = test['operation']['arguments']
for arg_name in list(arguments):
c2s = camel_to_snake(arg_name)
# PyMongo accepts sort as list of tuples. Asserting len=1
# because ordering dicts from JSON in 2.6 is unwieldy.
if arg_name == "sort":
sort_dict = arguments[arg_name]
assert len(sort_dict) == 1, 'test can only have 1 sort key'
arguments[arg_name] = list(iteritems(sort_dict))
# Named "key" instead not fieldName.
if arg_name == "fieldName":
arguments["key"] = arguments.pop(arg_name)
# Aggregate uses "batchSize", while find uses batch_size.
elif arg_name == "batchSize" and operation == "aggregate":
continue
# Requires boolean returnDocument.
elif arg_name == "returnDocument":
arguments[c2s] = arguments[arg_name] == "After"
else:
arguments[c2s] = arguments.pop(arg_name)
result = cmd(**arguments)
# Assert result is expected, excluding the $out aggregation test.
if not ignore_result:
check_result(test['outcome'].get('result'), result)
# Assert final state is expected.
expected_c = test['outcome'].get('collection')
if expected_c is not None:
expected_name = expected_c.get('name')
if expected_name is not None:
db_coll = self.db[expected_name]
else:
db_coll = self.db.test
self.assertEqual(list(db_coll.find()), expected_c['data'])
def _handle_option_deprecations(options):
"""Issue appropriate warnings when deprecated options are present in the
options dictionary. Removes deprecated option key, value pairs if the
options dictionary is found to also have the renamed option."""
undeprecated_options = _CaseInsensitiveDictionary()
for key, value in iteritems(options):
optname = str(key).lower()
if optname in URI_OPTIONS_DEPRECATION_MAP:
renamed_key = URI_OPTIONS_DEPRECATION_MAP[optname]
if renamed_key.lower() in options:
warnings.warn("Deprecated option '%s' ignored in favor of "
"'%s'." % (str(key), renamed_key))
continue
warnings.warn("Option '%s' is deprecated, use '%s' instead." % (
str(key), renamed_key))
undeprecated_options[str(key)] = value
return undeprecated_options
def validate_options(opts, warn=False):
"""Validates and normalizes options passed in a MongoDB URI.
Returns a new dictionary of validated and normalized options. If warn is
False then errors will be thrown for invalid options, otherwise they will
be ignored and a warning will be issued.
:Parameters:
- `opts`: A dict of MongoDB URI options.
- `warn` (optional): If ``True`` then warnigns will be logged and
invalid options will be ignored. Otherwise invalid options will
cause errors.
"""
if warn:
return get_validated_options(opts)
else:
return dict([_validate(opt, val) for opt, val in iteritems(opts)])
def run_operation(collection, test):
# Convert command from CamelCase to pymongo.collection method.
operation = camel_to_snake(test['operation']['name'])
cmd = getattr(collection, operation)
# Convert arguments to snake_case and handle special cases.
arguments = test['operation']['arguments']
options = arguments.pop("options", {})
for option_name in options:
arguments[camel_to_snake(option_name)] = options[option_name]
if operation == "bulk_write":
# Parse each request into a bulk write model.
requests = []
for request in arguments["requests"]:
bulk_model = camel_to_upper_camel(request["name"])
bulk_class = getattr(operations, bulk_model)
bulk_arguments = camel_to_snake_args(request["arguments"])
requests.append(bulk_class(**bulk_arguments))
arguments["requests"] = requests
else:
for arg_name in list(arguments):
c2s = camel_to_snake(arg_name)
# PyMongo accepts sort as list of tuples.
if arg_name == "sort":
sort_dict = arguments[arg_name]
arguments[arg_name] = list(iteritems(sort_dict))
# Named "key" instead not fieldName.
if arg_name == "fieldName":
arguments["key"] = arguments.pop(arg_name)
# Aggregate uses "batchSize", while find uses batch_size.
elif arg_name == "batchSize" and operation == "aggregate":
continue
# Requires boolean returnDocument.
elif arg_name == "returnDocument":
arguments[c2s] = arguments[arg_name] == "After"
else:
arguments[c2s] = arguments.pop(arg_name)
result = cmd(**arguments)
if operation == "aggregate":
if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]:
out = collection.database[arguments["pipeline"][-1]["$out"]]
return out.find()
return result
def _encode_dbref(name, value, check_keys, opts):
"""Encode bson.dbref.DBRef."""
buf = bytearray(b"\x03" + name + b"\x00\x00\x00\x00")
begin = len(buf) - 4
buf += _name_value_to_bson(b"$ref\x00",
value.collection, check_keys, opts)
buf += _name_value_to_bson(b"$id\x00",
value.id, check_keys, opts)
if value.database is not None:
buf += _name_value_to_bson(
b"$db\x00", value.database, check_keys, opts)
for key, val in iteritems(value._DBRef__kwargs):
buf += _element_to_bson(key, val, check_keys, opts)
buf += b"\x00"
buf[begin:begin + 4] = _PACK_INT(len(buf) - begin)
return bytes(buf)
def test_subclasses(self):
# make sure we can serialize subclasses of native Python types.
class _myint(int):
pass
class _myfloat(float):
pass
class _myunicode(text_type):
pass
d = {'a': _myint(42), 'b': _myfloat(63.9),
'c': _myunicode('hello world')
}
d2 = BSON.encode(d).decode()
for key, value in iteritems(d2):
orig_value = d[key]
orig_type = orig_value.__class__.__bases__[0]
self.assertEqual(type(value), orig_type)
self.assertEqual(value, orig_type(value))
def get_change_stream(client, scenario_def, test):
# Get target namespace on which to instantiate change stream
target = test["target"]
if target == "collection":
db = client.get_database(scenario_def["database_name"])
cs_target = db.get_collection(scenario_def["collection_name"])
elif target == "database":
cs_target = client.get_database(scenario_def["database_name"])
elif target == "client":
cs_target = client
else:
raise ValueError("Invalid target in spec")
# Construct change stream kwargs dict
cs_pipeline = test["changeStreamPipeline"]
options = test["changeStreamOptions"]
cs_options = {}
for key, value in iteritems(options):
cs_options[camel_to_snake(key)] = value
# Create and return change stream
return cs_target.watch(pipeline=cs_pipeline, **cs_options)
def simplify(case): # TODO this is a hack
if isinstance(case, SON) and "$ref" not in case:
simplified = SON(case) # make a copy!
if random.choice([True, False]):
# delete
simplified_keys = list(simplified)
if not len(simplified_keys):
return (False, case)
simplified.pop(random.choice(simplified_keys))
return (True, simplified)
else:
# simplify a value
simplified_items = list(iteritems(simplified))
if not len(simplified_items):
return (False, case)
(key, value) = random.choice(simplified_items)
(success, value) = simplify(value)
simplified[key] = value
return (success, success and simplified or case)
if isinstance(case, list):
simplified = list(case)
if random.choice([True, False]):
# delete
if not len(simplified):
return (False, case)
simplified.pop(random.randrange(len(simplified)))
return (True, simplified)
else:
# simplify an item
if not len(simplified):
return (False, case)
index = random.randrange(len(simplified))
(success, value) = simplify(simplified[index])
simplified[index] = value
return (success, success and simplified or case)
return (False, case)
'the client certificate and the private key')),
'ssl_keyfile':
('removed',
('Instead of using ssl_keyfile to specify the private keyfile, '
'use tlsCertificateKeyFile to pass a single file containing both '
'the client certificate and the private key')),
'ssl_pem_passphrase': ('renamed', 'tlsCertificateKeyFilePassword'),
'waitqueuemultiple':
('removed',
('Instead of using waitQueueMultiple to bound queuing, limit the size '
'of the thread pool in your application server'))
}
# Augment the option validator map with pymongo-specific option information.
URI_OPTIONS_VALIDATOR_MAP.update(NONSPEC_OPTIONS_VALIDATOR_MAP)
for optname, aliases in iteritems(URI_OPTIONS_ALIAS_MAP):
for alias in aliases:
if alias not in URI_OPTIONS_VALIDATOR_MAP:
URI_OPTIONS_VALIDATOR_MAP[alias] = (
URI_OPTIONS_VALIDATOR_MAP[optname])
# Map containing all URI option and keyword argument validators.
VALIDATORS = URI_OPTIONS_VALIDATOR_MAP.copy()
VALIDATORS.update(KW_VALIDATORS)
# List of timeout-related options.
TIMEOUT_OPTIONS = [
'connecttimeoutms',
'heartbeatfrequencyms',
'maxidletimems',
'maxstalenessseconds',
def run_operation(self, sessions, collection, operation):
name = camel_to_snake(operation['name'])
if name == 'run_command':
name = 'command'
self.transaction_test_debug(name)
def parse_options(opts):
if 'readPreference' in opts:
opts['read_preference'] = parse_read_preference(
opts.pop('readPreference'))
if 'writeConcern' in opts:
opts['write_concern'] = WriteConcern(
**dict(opts.pop('writeConcern')))
if 'readConcern' in opts:
opts['read_concern'] = ReadConcern(
**dict(opts.pop('readConcern')))
return opts
database = collection.database
collection = database.get_collection(collection.name)
if 'collectionOptions' in operation:
collection = collection.with_options(
**dict(parse_options(operation['collectionOptions'])))
objects = {'database': database, 'collection': collection}
objects.update(sessions)
obj = objects[operation['object']]
# Combine arguments with options and handle special cases.
arguments = operation.get('arguments', {})
arguments.update(arguments.pop("options", {}))
parse_options(arguments)
cmd = getattr(obj, name)
for arg_name in list(arguments):
c2s = camel_to_snake(arg_name)
# PyMongo accepts sort as list of tuples. Asserting len=1
# because ordering dicts from JSON in 2.6 is unwieldy.
if arg_name == "sort":
sort_dict = arguments[arg_name]
assert len(sort_dict) == 1, 'test can only have 1 sort key'
arguments[arg_name] = list(iteritems(sort_dict))
# Named "key" instead not fieldName.
if arg_name == "fieldName":
arguments["key"] = arguments.pop(arg_name)
# Aggregate uses "batchSize", while find uses batch_size.
elif arg_name == "batchSize" and name == "aggregate":
continue
# Requires boolean returnDocument.
elif arg_name == "returnDocument":
arguments[c2s] = arguments[arg_name] == "After"
elif c2s == "requests":
# Parse each request into a bulk write model.
requests = []
for request in arguments["requests"]:
bulk_model = camel_to_upper_camel(request["name"])
bulk_class = getattr(operations, bulk_model)
bulk_arguments = camel_to_snake_args(request["arguments"])
requests.append(bulk_class(**dict(bulk_arguments)))
arguments["requests"] = requests
elif arg_name == "session":
arguments['session'] = sessions[arguments['session']]
elif name == 'command' and arg_name == 'command':
# Ensure the first key is the command name.
ordered_command = SON([(operation['command_name'], 1)])
ordered_command.update(arguments['command'])
arguments['command'] = ordered_command
else:
arguments[c2s] = arguments.pop(arg_name)
result = cmd(**dict(arguments))
if name == "aggregate":
if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]:
# Read from the primary to ensure causal consistency.
out = collection.database.get_collection(
arguments["pipeline"][-1]["$out"],
read_preference=ReadPreference.PRIMARY)
return out.find()
if isinstance(result, Cursor) or isinstance(result, CommandCursor):
return list(result)
return result
def run_operation(self, sessions, collection, operation):
name = camel_to_snake(operation['name'])
if name == 'run_command':
name = 'command'
self.transaction_test_debug(name)
def parse_options(opts):
if 'readPreference' in opts:
opts['read_preference'] = parse_read_preference(
opts.pop('readPreference'))
if 'writeConcern' in opts:
opts['write_concern'] = WriteConcern(
**dict(opts.pop('writeConcern')))
if 'readConcern' in opts:
opts['read_concern'] = ReadConcern(
**dict(opts.pop('readConcern')))
return opts
database = collection.database
collection = database.get_collection(collection.name)
if 'collectionOptions' in operation:
collection = collection.with_options(
**dict(parse_options(operation['collectionOptions'])))
objects = {'database': database, 'collection': collection}
objects.update(sessions)
obj = objects[operation['object']]
# Combine arguments with options and handle special cases.
arguments = operation.get('arguments', {})
arguments.update(arguments.pop("options", {}))
parse_options(arguments)
cmd = getattr(obj, name)
for arg_name in list(arguments):
c2s = camel_to_snake(arg_name)
# PyMongo accepts sort as list of tuples.
if arg_name == "sort":
sort_dict = arguments[arg_name]
arguments[arg_name] = list(iteritems(sort_dict))
# Named "key" instead not fieldName.
if arg_name == "fieldName":
arguments["key"] = arguments.pop(arg_name)
# Aggregate uses "batchSize", while find uses batch_size.
elif arg_name == "batchSize" and name == "aggregate":
continue
# Requires boolean returnDocument.
elif arg_name == "returnDocument":
arguments[c2s] = arguments[arg_name] == "After"
elif c2s == "requests":
# Parse each request into a bulk write model.
requests = []
for request in arguments["requests"]:
bulk_model = camel_to_upper_camel(request["name"])
bulk_class = getattr(operations, bulk_model)
bulk_arguments = camel_to_snake_args(request["arguments"])
requests.append(bulk_class(**dict(bulk_arguments)))
arguments["requests"] = requests
elif arg_name == "session":
arguments['session'] = sessions[arguments['session']]
elif name == 'command' and arg_name == 'command':
# Ensure the first key is the command name.
ordered_command = SON([(operation['command_name'], 1)])
ordered_command.update(arguments['command'])
arguments['command'] = ordered_command
else:
arguments[c2s] = arguments.pop(arg_name)
result = cmd(**dict(arguments))
if name == "aggregate":
if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]:
# Read from the primary to ensure causal consistency.
out = collection.database.get_collection(
arguments["pipeline"][-1]["$out"],
read_preference=ReadPreference.PRIMARY)
return out.find()
if isinstance(result, Cursor) or isinstance(result, CommandCursor):
return list(result)
return result
def authenticate(self,
name=None,
password=None,
source=None,
mechanism='DEFAULT',
**kwargs):
"""**DEPRECATED**: Authenticate to use this database.
Authentication lasts for the life of the underlying client
instance, or until :meth:`logout` is called.
Raises :class:`TypeError` if (required) `name`, (optional) `password`,
or (optional) `source` is not an instance of :class:`basestring`
(:class:`str` in python 3).
.. note::
- This method authenticates the current connection, and
will also cause all new :class:`~socket.socket` connections
in the underlying client instance to be authenticated automatically.
- Authenticating more than once on the same database with different
credentials is not supported. You must call :meth:`logout` before
authenticating with new credentials.
- When sharing a client instance between multiple threads, all
threads will share the authentication. If you need different
authentication profiles for different purposes you must use
distinct client instances.
:Parameters:
- `name`: the name of the user to authenticate. Optional when
`mechanism` is MONGODB-X509 and the MongoDB server version is
>= 3.4.
- `password` (optional): the password of the user to authenticate.
Not used with GSSAPI or MONGODB-X509 authentication.
- `source` (optional): the database to authenticate on. If not
specified the current database is used.
- `mechanism` (optional): See
:data:`~pymongo.auth.MECHANISMS` for options.
By default, use SCRAM-SHA-1 with MongoDB 3.0 and later,
MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
- `authMechanismProperties` (optional): Used to specify
authentication mechanism specific options. To specify the service
name for GSSAPI authentication pass
authMechanismProperties='SERVICE_NAME:<service name>'
.. versionchanged:: 3.5
Deprecated. Authenticating multiple users conflicts with support for
logical sessions in MongoDB 3.6. To authenticate as multiple users,
create multiple instances of MongoClient.
.. versionadded:: 2.8
Use SCRAM-SHA-1 with MongoDB 3.0 and later.
.. versionchanged:: 2.5
Added the `source` and `mechanism` parameters. :meth:`authenticate`
now raises a subclass of :class:`~pymongo.errors.PyMongoError` if
authentication fails due to invalid credentials or configuration
issues.
.. mongodoc:: authenticate
"""
if name is not None and not isinstance(name, string_type):
raise TypeError("name must be an "
"instance of %s" % (string_type.__name__, ))
if password is not None and not isinstance(password, string_type):
raise TypeError("password must be an "
"instance of %s" % (string_type.__name__, ))
if source is not None and not isinstance(source, string_type):
raise TypeError("source must be an "
"instance of %s" % (string_type.__name__, ))
common.validate_auth_mechanism('mechanism', mechanism)
validated_options = {}
for option, value in iteritems(kwargs):
normalized, val = common.validate_auth_option(option, value)
validated_options[normalized] = val
credentials = auth._build_credentials_tuple(mechanism, source
or self.name, name,
password,
validated_options)
self.client._cache_credentials(self.name, credentials, connect=True)
return True
def run_operation(self, sessions, collection, operation):
original_collection = collection
name = camel_to_snake(operation['name'])
if name == 'run_command':
name = 'command'
elif name == 'download_by_name':
name = 'open_download_stream_by_name'
elif name == 'download':
name = 'open_download_stream'
database = collection.database
collection = database.get_collection(collection.name)
if 'collectionOptions' in operation:
collection = collection.with_options(
**self.parse_options(operation['collectionOptions']))
object_name = self.get_object_name(operation)
if object_name == 'gridfsbucket':
# Only create the GridFSBucket when we need it (for the gridfs
# retryable reads tests).
obj = GridFSBucket(database,
bucket_name=collection.name,
disable_md5=True)
else:
objects = {
'client': database.client,
'database': database,
'collection': collection,
'testRunner': self
}
objects.update(sessions)
obj = objects[object_name]
# Combine arguments with options and handle special cases.
arguments = operation.get('arguments', {})
arguments.update(arguments.pop("options", {}))
self.parse_options(arguments)
cmd = getattr(obj, name)
for arg_name in list(arguments):
c2s = camel_to_snake(arg_name)
# PyMongo accepts sort as list of tuples.
if arg_name == "sort":
sort_dict = arguments[arg_name]
arguments[arg_name] = list(iteritems(sort_dict))
# Named "key" instead not fieldName.
if arg_name == "fieldName":
arguments["key"] = arguments.pop(arg_name)
# Aggregate uses "batchSize", while find uses batch_size.
elif ((arg_name == "batchSize" or arg_name == "allowDiskUse")
and name == "aggregate"):
continue
# Requires boolean returnDocument.
elif arg_name == "returnDocument":
arguments[c2s] = arguments.pop(arg_name) == "After"
elif c2s == "requests":
# Parse each request into a bulk write model.
requests = []
for request in arguments["requests"]:
bulk_model = camel_to_upper_camel(request["name"])
bulk_class = getattr(operations, bulk_model)
bulk_arguments = camel_to_snake_args(request["arguments"])
requests.append(bulk_class(**dict(bulk_arguments)))
arguments["requests"] = requests
elif arg_name == "session":
arguments['session'] = sessions[arguments['session']]
elif (name in ('command', 'run_admin_command')
and arg_name == 'command'):
# Ensure the first key is the command name.
ordered_command = SON([(operation['command_name'], 1)])
ordered_command.update(arguments['command'])
arguments['command'] = ordered_command
elif name == 'open_download_stream' and arg_name == 'id':
arguments['file_id'] = arguments.pop(arg_name)
elif name != 'find' and c2s == 'max_time_ms':
# find is the only method that accepts snake_case max_time_ms.
# All other methods take kwargs which must use the server's
# camelCase maxTimeMS. See PYTHON-1855.
arguments['maxTimeMS'] = arguments.pop('max_time_ms')
elif name == 'with_transaction' and arg_name == 'callback':
callback_ops = arguments[arg_name]['operations']
arguments['callback'] = lambda _: self.run_operations(
sessions,
original_collection,
copy.deepcopy(callback_ops),
in_with_transaction=True)
elif name == 'drop_collection' and arg_name == 'collection':
arguments['name_or_collection'] = arguments.pop(arg_name)
elif name == 'create_collection' and arg_name == 'collection':
arguments['name'] = arguments.pop(arg_name)
elif name == 'create_index' and arg_name == 'keys':
arguments['keys'] = list(arguments.pop(arg_name).items())
elif name == 'drop_index' and arg_name == 'name':
arguments['index_or_name'] = arguments.pop(arg_name)
else:
arguments[c2s] = arguments.pop(arg_name)
if name == 'run_on_thread':
args = {'sessions': sessions, 'collection': collection}
args.update(arguments)
arguments = args
result = cmd(**dict(arguments))
if name == "aggregate":
if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]:
# Read from the primary to ensure causal consistency.
out = collection.database.get_collection(
arguments["pipeline"][-1]["$out"],
read_preference=ReadPreference.PRIMARY)
return out.find()
if name == "map_reduce":
if isinstance(result, dict) and 'results' in result:
return result['results']
if 'download' in name:
result = Binary(result.read())
if isinstance(result, Cursor) or isinstance(result, CommandCursor):
return list(result)
return result
def _encode_mapping(name, value, check_keys, opts):
"""Encode a mapping type."""
data = b"".join([_element_to_bson(key, val, check_keys, opts)
for key, val in iteritems(value)])
return b"\x03" + name + _PACK_INT(len(data) + 5) + data + b"\x00"
def run_operation(self, sessions, collection, operation):
session = None
name = camel_to_snake(operation['name'])
self.transaction_test_debug(name)
session_name = operation['arguments'].pop('session', None)
if session_name:
session = sessions[session_name]
# Combine arguments with options and handle special cases.
arguments = operation['arguments']
arguments.update(arguments.pop("options", {}))
pref = write_c = read_c = None
if 'readPreference' in arguments:
pref = parse_read_preference(arguments.pop('readPreference'))
if 'writeConcern' in arguments:
write_c = WriteConcern(**dict(arguments.pop('writeConcern')))
if 'readConcern' in arguments:
read_c = ReadConcern(**dict(arguments.pop('readConcern')))
if name == 'start_transaction':
cmd = partial(session.start_transaction,
write_concern=write_c,
read_concern=read_c,
read_preference=pref)
elif name in ('commit_transaction', 'abort_transaction'):
cmd = getattr(session, name)
else:
collection = collection.with_options(write_concern=write_c,
read_concern=read_c,
read_preference=pref)
cmd = getattr(collection, name)
arguments['session'] = session
for arg_name in list(arguments):
c2s = camel_to_snake(arg_name)
# PyMongo accepts sort as list of tuples. Asserting len=1
# because ordering dicts from JSON in 2.6 is unwieldy.
if arg_name == "sort":
sort_dict = arguments[arg_name]
assert len(sort_dict) == 1, 'test can only have 1 sort key'
arguments[arg_name] = list(iteritems(sort_dict))
# Named "key" instead not fieldName.
if arg_name == "fieldName":
arguments["key"] = arguments.pop(arg_name)
# Aggregate uses "batchSize", while find uses batch_size.
elif arg_name == "batchSize" and name == "aggregate":
continue
# Requires boolean returnDocument.
elif arg_name == "returnDocument":
arguments[c2s] = arguments[arg_name] == "After"
elif c2s == "requests":
# Parse each request into a bulk write model.
requests = []
for request in arguments["requests"]:
bulk_model = camel_to_upper_camel(request["name"])
bulk_class = getattr(operations, bulk_model)
bulk_arguments = camel_to_snake_args(request["arguments"])
requests.append(bulk_class(**dict(bulk_arguments)))
arguments["requests"] = requests
else:
arguments[c2s] = arguments.pop(arg_name)
result = cmd(**dict(arguments))
if name == "aggregate":
if arguments["pipeline"] and "$out" in arguments["pipeline"][-1]:
# Read from the primary to ensure causal consistency.
out = collection.database.get_collection(
arguments["pipeline"][-1]["$out"],
read_preference=ReadPreference.PRIMARY)
return out.find()
if isinstance(result, Cursor) or isinstance(result, CommandCursor):
return list(result)
return result
def items(self):
"""Lazily decode and iterate elements in this document."""
return iteritems(self.__inflated)
def validate_collection(self,
name_or_collection,
scandata=False,
full=False):
"""Validate a collection.
Returns a dict of validation info. Raises CollectionInvalid if
validation fails.
With MongoDB < 1.9 the result dict will include a `result` key
with a string value that represents the validation results. With
MongoDB >= 1.9 the `result` key no longer exists and the results
are split into individual fields in the result dict.
:Parameters:
- `name_or_collection`: A Collection object or the name of a
collection to validate.
- `scandata`: Do extra checks beyond checking the overall
structure of the collection.
- `full`: Have the server do a more thorough scan of the
collection. Use with `scandata` for a thorough scan
of the structure of the collection and the individual
documents. Ignored in MongoDB versions before 1.9.
"""
name = name_or_collection
if isinstance(name, Collection):
name = name.name
if not isinstance(name, string_type):
raise TypeError("name_or_collection must be an instance of "
"%s or Collection" % (string_type.__name__, ))
result = self.command("validate",
_unicode(name),
scandata=scandata,
full=full)
valid = True
# Pre 1.9 results
if "result" in result:
info = result["result"]
if info.find("exception") != -1 or info.find("corrupt") != -1:
raise CollectionInvalid("%s invalid: %s" % (name, info))
# Sharded results
elif "raw" in result:
for _, res in iteritems(result["raw"]):
if "result" in res:
info = res["result"]
if (info.find("exception") != -1
or info.find("corrupt") != -1):
raise CollectionInvalid("%s invalid: "
"%s" % (name, info))
elif not res.get("valid", False):
valid = False
break
# Post 1.9 non-sharded results.
elif not result.get("valid", False):
valid = False
if not valid:
raise CollectionInvalid("%s invalid: %r" % (name, result))
return result
def _create_entity(self, entity_spec):
if len(entity_spec) != 1:
self._test_class.fail(
"Entity spec %s did not contain exactly one top-level key" %
(entity_spec, ))
entity_type, spec = next(iteritems(entity_spec))
if entity_type == 'client':
kwargs = {}
observe_events = spec.get('observeEvents', [])
ignore_commands = spec.get('ignoreCommandMonitoringEvents', [])
if len(observe_events) or len(ignore_commands):
ignore_commands = [cmd.lower() for cmd in ignore_commands]
listener = EventListenerUtil(observe_events, ignore_commands)
self._listeners[spec['id']] = listener
kwargs['event_listeners'] = [listener]
if client_context.is_mongos and spec.get('useMultipleMongoses'):
kwargs['h'] = client_context.mongos_seeds()
kwargs.update(spec.get('uriOptions', {}))
server_api = spec.get('serverApi')
if server_api:
kwargs['server_api'] = ServerApi(
server_api['version'],
strict=server_api.get('strict'),
deprecation_errors=server_api.get('deprecationErrors'))
client = rs_or_single_client(**kwargs)
self[spec['id']] = client
self._test_class.addCleanup(client.close)
return
elif entity_type == 'database':
client = self[spec['client']]
if not isinstance(client, MongoClient):
self._test_class.fail(
'Expected entity %s to be of type MongoClient, got %s' %
(spec['client'], type(client)))
options = parse_collection_or_database_options(
spec.get('databaseOptions', {}))
self[spec['id']] = client.get_database(spec['databaseName'],
**options)
return
elif entity_type == 'collection':
database = self[spec['database']]
if not isinstance(database, Database):
self._test_class.fail(
'Expected entity %s to be of type Database, got %s' %
(spec['database'], type(database)))
options = parse_collection_or_database_options(
spec.get('collectionOptions', {}))
self[spec['id']] = database.get_collection(spec['collectionName'],
**options)
return
elif entity_type == 'session':
client = self[spec['client']]
if not isinstance(client, MongoClient):
self._test_class.fail(
'Expected entity %s to be of type MongoClient, got %s' %
(spec['client'], type(client)))
opts = camel_to_snake_args(spec.get('sessionOptions', {}))
if 'default_transaction_options' in opts:
txn_opts = parse_spec_options(
opts['default_transaction_options'])
txn_opts = TransactionOptions(**txn_opts)
opts = copy.deepcopy(opts)
opts['default_transaction_options'] = txn_opts
session = client.start_session(**dict(opts))
self[spec['id']] = session
self._session_lsids[spec['id']] = copy.deepcopy(session.session_id)
self._test_class.addCleanup(session.end_session)
return
elif entity_type == 'bucket':
# TODO: implement the 'bucket' entity type
self._test_class.skipTest(
'GridFS is not currently supported (PYTHON-2459)')
self._test_class.fail('Unable to create entity of unknown type %s' %
(entity_type, ))
# Map from deprecated URI option names to the updated option names.
# Case is preserved for updated option names as they are part of user warnings.
URI_OPTIONS_DEPRECATION_MAP = {
'j': 'journal',
'wtimeout': 'wTimeoutMS',
'ssl_cert_reqs': 'tlsAllowInvalidCertificates',
'ssl_match_hostname': 'tlsAllowInvalidHostnames',
'ssl_crlfile': 'tlsCRLFile',
'ssl_ca_certs': 'tlsCAFile',
'ssl_pem_passphrase': 'tlsCertificateKeyFilePassword',
}
# Augment the option validator map with pymongo-specific option information.
URI_OPTIONS_VALIDATOR_MAP.update(NONSPEC_OPTIONS_VALIDATOR_MAP)
for optname, aliases in iteritems(URI_OPTIONS_ALIAS_MAP):
for alias in aliases:
if alias not in URI_OPTIONS_VALIDATOR_MAP:
URI_OPTIONS_VALIDATOR_MAP[alias] = (
URI_OPTIONS_VALIDATOR_MAP[optname])
# Map containing all URI option and keyword argument validators.
VALIDATORS = URI_OPTIONS_VALIDATOR_MAP.copy()
VALIDATORS.update(KW_VALIDATORS)
# List of timeout-related options.
TIMEOUT_OPTIONS = [
'connecttimeoutms',
'heartbeatfrequencyms',
'maxidletimems',
'maxstalenessseconds',
def prepare_spec_arguments(spec, arguments, opname, entity_map,
with_txn_callback):
for arg_name in list(arguments):
c2s = camel_to_snake(arg_name)
# PyMongo accepts sort as list of tuples.
if arg_name == "sort":
sort_dict = arguments[arg_name]
arguments[arg_name] = list(iteritems(sort_dict))
# Named "key" instead not fieldName.
if arg_name == "fieldName":
arguments["key"] = arguments.pop(arg_name)
# Aggregate uses "batchSize", while find uses batch_size.
elif ((arg_name == "batchSize" or arg_name == "allowDiskUse")
and opname == "aggregate"):
continue
# Requires boolean returnDocument.
elif arg_name == "returnDocument":
arguments[c2s] = getattr(ReturnDocument,
arguments.pop(arg_name).upper())
elif c2s == "requests":
# Parse each request into a bulk write model.
requests = []
for request in arguments["requests"]:
if 'name' in request:
# CRUD v2 format
bulk_model = camel_to_upper_camel(request["name"])
bulk_class = getattr(operations, bulk_model)
bulk_arguments = camel_to_snake_args(request["arguments"])
else:
# Unified test format
bulk_model, spec = next(iteritems(request))
bulk_class = getattr(operations,
camel_to_upper_camel(bulk_model))
bulk_arguments = camel_to_snake_args(spec)
requests.append(bulk_class(**dict(bulk_arguments)))
arguments["requests"] = requests
elif arg_name == "session":
arguments['session'] = entity_map[arguments['session']]
elif (opname in ('command', 'run_admin_command')
and arg_name == 'command'):
# Ensure the first key is the command name.
ordered_command = SON([(spec['command_name'], 1)])
ordered_command.update(arguments['command'])
arguments['command'] = ordered_command
elif opname == 'open_download_stream' and arg_name == 'id':
arguments['file_id'] = arguments.pop(arg_name)
elif opname != 'find' and c2s == 'max_time_ms':
# find is the only method that accepts snake_case max_time_ms.
# All other methods take kwargs which must use the server's
# camelCase maxTimeMS. See PYTHON-1855.
arguments['maxTimeMS'] = arguments.pop('max_time_ms')
elif opname == 'with_transaction' and arg_name == 'callback':
if 'operations' in arguments[arg_name]:
# CRUD v2 format
callback_ops = arguments[arg_name]['operations']
else:
# Unified test format
callback_ops = arguments[arg_name]
arguments['callback'] = lambda _: with_txn_callback(
copy.deepcopy(callback_ops))
elif opname == 'drop_collection' and arg_name == 'collection':
arguments['name_or_collection'] = arguments.pop(arg_name)
elif opname == 'create_collection' and arg_name == 'collection':
arguments['name'] = arguments.pop(arg_name)
elif opname == 'create_index' and arg_name == 'keys':
arguments['keys'] = list(arguments.pop(arg_name).items())
elif opname == 'drop_index' and arg_name == 'name':
arguments['index_or_name'] = arguments.pop(arg_name)
else:
arguments[c2s] = arguments.pop(arg_name)
def authenticate(self, name, password=None,
source=None, mechanism='DEFAULT', **kwargs):
"""Authenticate to use this database.
Authentication lasts for the life of the underlying client
instance, or until :meth:`logout` is called.
Raises :class:`TypeError` if (required) `name`, (optional) `password`,
or (optional) `source` is not an instance of :class:`basestring`
(:class:`str` in python 3).
.. note::
- This method authenticates the current connection, and
will also cause all new :class:`~socket.socket` connections
in the underlying client instance to be authenticated automatically.
- Authenticating more than once on the same database with different
credentials is not supported. You must call :meth:`logout` before
authenticating with new credentials.
- When sharing a client instance between multiple threads, all
threads will share the authentication. If you need different
authentication profiles for different purposes you must use
distinct client instances.
:Parameters:
- `name`: the name of the user to authenticate.
- `password` (optional): the password of the user to authenticate.
Not used with GSSAPI or MONGODB-X509 authentication.
- `source` (optional): the database to authenticate on. If not
specified the current database is used.
- `mechanism` (optional): See
:data:`~pymongo.auth.MECHANISMS` for options.
By default, use SCRAM-SHA-1 with MongoDB 3.0 and later,
MONGODB-CR (MongoDB Challenge Response protocol) for older servers.
- `authMechanismProperties` (optional): Used to specify
authentication mechanism specific options. To specify the service
name for GSSAPI authentication pass
authMechanismProperties='SERVICE_NAME:<service name>'
.. versionadded:: 2.8
Use SCRAM-SHA-1 with MongoDB 3.0 and later.
.. versionchanged:: 2.5
Added the `source` and `mechanism` parameters. :meth:`authenticate`
now raises a subclass of :class:`~pymongo.errors.PyMongoError` if
authentication fails due to invalid credentials or configuration
issues.
.. mongodoc:: authenticate
"""
if not isinstance(name, string_type):
raise TypeError("name must be an "
"instance of %s" % (string_type.__name__,))
if password is not None and not isinstance(password, string_type):
raise TypeError("password must be an "
"instance of %s" % (string_type.__name__,))
if source is not None and not isinstance(source, string_type):
raise TypeError("source must be an "
"instance of %s" % (string_type.__name__,))
common.validate_auth_mechanism('mechanism', mechanism)
validated_options = {}
for option, value in iteritems(kwargs):
normalized, val = common.validate_auth_option(option, value)
validated_options[normalized] = val
credentials = auth._build_credentials_tuple(
mechanism,
source or self.name,
name,
password,
validated_options)
self.client._cache_credentials(
self.name,
credentials,
connect=True)
return True
def operation(self, **operations):
"""Get the object that represent the `operations` in a Mongo query.
"""
return {('$' + operation): value
for operation, value in iteritems(operations)}
def items(self):
"""Lazily decode and iterate elements in this document."""
return iteritems(self.__inflated)
