def __init__(self):
"""Create a client and grab essential information from the server."""
self.connection_attempts = []
self.connected = False
self.w = None
self.nodes = set()
self.replica_set_name = None
self.cmd_line = None
self.server_status = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.server_parameters = None
self.is_mongos = False
self.mongoses = []
self.is_rs = False
self.has_ipv6 = False
self.tls = False
self.ssl_certfile = False
self.server_is_resolvable = is_server_resolvable()
self.default_client_options = {}
self.sessions_enabled = False
self.client = None
self.conn_lock = threading.Lock()
self.is_data_lake = False
if COMPRESSORS:
self.default_client_options["compressors"] = COMPRESSORS
if MONGODB_API_VERSION:
server_api = ServerApi(MONGODB_API_VERSION)
self.default_client_options["server_api"] = server_api
def is_run_on_requirement_satisfied(requirement):
topology_satisfied = True
req_topologies = requirement.get('topologies')
if req_topologies:
topology_satisfied = client_context.is_topology_type(req_topologies)
min_version_satisfied = True
req_min_server_version = requirement.get('minServerVersion')
if req_min_server_version:
min_version_satisfied = Version.from_string(
req_min_server_version) <= client_context.version
max_version_satisfied = True
req_max_server_version = requirement.get('maxServerVersion')
if req_max_server_version:
max_version_satisfied = Version.from_string(
req_max_server_version) >= client_context.version
params_satisfied = True
params = requirement.get('serverParameters')
if params:
for param, val in params.items():
if param not in client_context.server_parameters:
params_satisfied = False
elif client_context.server_parameters[param] != val:
params_satisfied = False
return (topology_satisfied and min_version_satisfied
and max_version_satisfied and params_satisfied)
def is_run_on_requirement_satisfied(requirement):
topology_satisfied = True
req_topologies = requirement.get('topologies')
if req_topologies:
topology_satisfied = client_context.is_topology_type(req_topologies)
server_version = Version(*client_context.version[:3])
min_version_satisfied = True
req_min_server_version = requirement.get('minServerVersion')
if req_min_server_version:
min_version_satisfied = Version.from_string(
req_min_server_version) <= server_version
max_version_satisfied = True
req_max_server_version = requirement.get('maxServerVersion')
if req_max_server_version:
max_version_satisfied = Version.from_string(
req_max_server_version) >= server_version
serverless = requirement.get('serverless')
if serverless == "require":
serverless_satisfied = client_context.serverless
elif serverless == "forbid":
serverless_satisfied = not client_context.serverless
else: # unset or "allow"
serverless_satisfied = True
params_satisfied = True
params = requirement.get('serverParameters')
if params:
for param, val in params.items():
if param not in client_context.server_parameters:
params_satisfied = False
elif client_context.server_parameters[param] != val:
params_satisfied = False
auth_satisfied = True
req_auth = requirement.get('auth')
if req_auth is not None:
if req_auth:
auth_satisfied = client_context.auth_enabled
else:
auth_satisfied = not client_context.auth_enabled
return (topology_satisfied and min_version_satisfied
and max_version_satisfied and serverless_satisfied
and params_satisfied and auth_satisfied)
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connection_attempts = []
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.cmd_line = None
self.server_status = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.mongoses = []
self.is_rs = False
self.has_ipv6 = False
self.ssl = False
self.ssl_cert_none = False
self.ssl_certfile = False
self.server_is_resolvable = is_server_resolvable()
self.default_client_options = {}
self.sessions_enabled = False
self.client = None
self.conn_lock = threading.Lock()
if COMPRESSORS:
self.default_client_options["compressors"] = COMPRESSORS
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connection_attempts = []
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.cmd_line = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.is_rs = False
self.has_ipv6 = False
self.ssl = False
self.ssl_cert_none = False
self.ssl_certfile = False
self.server_is_resolvable = is_server_resolvable()
self.default_client_options = {}
self.sessions_enabled = False
self.client = None
self.conn_lock = threading.Lock()
if COMPRESSORS:
self.default_client_options["compressors"] = COMPRESSORS
def require_version_max(self, *ver):
"""Run a test only if the server version is at most ``version``."""
other_version = Version(*ver)
return self.require(
lambda: self.version <= other_version,
"Server version must be at most %s" % str(other_version),
)
def generate_test_classes(test_path,
module=__name__,
class_name_prefix='',
expected_failures=[],
bypass_test_generation_errors=False,
**kwargs):
"""Method for generating test classes. Returns a dictionary where keys are
the names of test classes and values are the test class objects."""
test_klasses = {}
def test_base_class_factory(test_spec):
"""Utility that creates the base class to use for test generation.
This is needed to ensure that cls.TEST_SPEC is appropriately set when
the metaclass __init__ is invoked."""
class SpecTestBase(with_metaclass(UnifiedSpecTestMeta)):
TEST_SPEC = test_spec
EXPECTED_FAILURES = expected_failures
return SpecTestBase
for dirpath, _, filenames in os.walk(test_path):
dirname = os.path.split(dirpath)[-1]
for filename in filenames:
fpath = os.path.join(dirpath, filename)
with open(fpath) as scenario_stream:
# Use tz_aware=False to match how CodecOptions decodes
# dates.
opts = json_util.JSONOptions(tz_aware=False)
scenario_def = json_util.loads(scenario_stream.read(),
json_options=opts)
test_type = os.path.splitext(filename)[0]
snake_class_name = 'Test%s_%s_%s' % (
class_name_prefix, dirname.replace(
'-', '_'), test_type.replace('-', '_').replace('.', '_'))
class_name = snake_to_camel(snake_class_name)
try:
schema_version = Version.from_string(
scenario_def['schemaVersion'])
mixin_class = _SCHEMA_VERSION_MAJOR_TO_MIXIN_CLASS.get(
schema_version[0])
if mixin_class is None:
raise ValueError(
"test file '%s' has unsupported schemaVersion '%s'" %
(fpath, schema_version))
module_dict = {'__module__': module}
module_dict.update(kwargs)
test_klasses[class_name] = type(class_name, (
mixin_class,
test_base_class_factory(scenario_def),
), module_dict)
except Exception:
if bypass_test_generation_errors:
continue
raise
return test_klasses
def remove_all_users(db):
if Version.from_client(db.client).at_least(2, 5, 3, -1):
db.command("dropAllUsersFromDatabase", 1,
writeConcern={"w": client_context.w})
else:
db = db.client.get_database(
db.name, write_concern=WriteConcern(w=client_context.w))
db.system.users.delete_many({})
def remove_all_users(db):
if Version.from_client(db.client).at_least(2, 5, 3, -1):
db.command("dropAllUsersFromDatabase", 1,
writeConcern={"w": client_context.w})
else:
db = db.client.get_database(
db.name, write_concern=WriteConcern(w=client_context.w))
db.system.users.delete_many({})
def setUpClass(cls):
super(MotorTransactionTest, cls).setUpClass()
if not env.sessions_enabled:
raise SkipTest("Sessions not supported")
if not env.is_replica_set:
raise SkipTest("Requires a replica set")
if env.version < Version(3, 7):
raise SkipTest("Requires MongoDB 3.7+")
def setUp(self):
super(TestCommandAndReadPreference, self).setUp()
self.c = ReadPrefTester(
'%s:%s' % (host, port),
replicaSet=self.name,
# Ignore round trip times, to test ReadPreference modes only.
localThresholdMS=1000*1000)
if client_context.auth_enabled:
self.c.admin.authenticate(db_user, db_pwd)
self.client_version = Version.from_client(self.c)
self.addCleanup(self.c.drop_database, 'pymongo_test')
def setUp(self):
super(TestCommandAndReadPreference, self).setUp()
self.c = ReadPrefTester(
'%s:%s' % (host, port),
replicaSet=self.name,
# Ignore round trip times, to test ReadPreference modes only.
localThresholdMS=1000*1000)
if client_context.auth_enabled:
self.c.admin.authenticate(db_user, db_pwd)
self.client_version = Version.from_client(self.c)
self.addCleanup(self.c.drop_database, 'pymongo_test')
def is_run_on_requirement_satisfied(requirement):
topology_satisfied = True
req_topologies = requirement.get('topologies')
if req_topologies:
topology_satisfied = client_context.is_topology_type(req_topologies)
min_version_satisfied = True
req_min_server_version = requirement.get('minServerVersion')
if req_min_server_version:
min_version_satisfied = Version.from_string(
req_min_server_version) <= client_context.version
max_version_satisfied = True
req_max_server_version = requirement.get('maxServerVersion')
if req_max_server_version:
max_version_satisfied = Version.from_string(
req_max_server_version) >= client_context.version
return (topology_satisfied and min_version_satisfied
and max_version_satisfied)
def setUpClass(cls):
super(TestCommandAndReadPreference, cls).setUpClass()
cls.c = ReadPrefTester(
client_context.pair,
# Ignore round trip times, to test ReadPreference modes only.
localThresholdMS=1000*1000)
cls.client_version = Version.from_client(cls.c)
# mapReduce fails if the collection does not exist.
coll = cls.c.pymongo_test.get_collection(
'test', write_concern=WriteConcern(w=client_context.w))
coll.insert_one({})
def setUpClass(cls):
super(TestCommandAndReadPreference, cls).setUpClass()
cls.c = ReadPrefTester(
client_context.pair,
replicaSet=cls.name,
# Ignore round trip times, to test ReadPreference modes only.
localThresholdMS=1000*1000)
if client_context.auth_enabled:
cls.c.admin.authenticate(db_user, db_pwd)
cls.client_version = Version.from_client(cls.c)
# mapReduce and group fail with no collection
coll = cls.c.pymongo_test.get_collection(
'test', write_concern=WriteConcern(w=cls.w))
coll.insert_one({})
def setUpClass(cls):
super(TestCommandAndReadPreference, cls).setUpClass()
cls.c = ReadPrefTester(
client_context.pair,
replicaSet=cls.name,
# Ignore round trip times, to test ReadPreference modes only.
localThresholdMS=1000 * 1000)
if client_context.auth_enabled:
cls.c.admin.authenticate(db_user, db_pwd)
cls.client_version = Version.from_client(cls.c)
# mapReduce and group fail with no collection
coll = cls.c.pymongo_test.get_collection(
'test', write_concern=WriteConcern(w=cls.w))
coll.insert_one({})
def setUp(self):
super(UnifiedSpecTestMixinV1, self).setUp()
# process schemaVersion
# note: we check major schema version during class generation
# note: we do this here because we cannot run assertions in setUpClass
version = Version.from_string(self.TEST_SPEC['schemaVersion'])
self.assertLessEqual(
version, self.SCHEMA_VERSION,
'expected schema version %s or lower, got %s' %
(self.SCHEMA_VERSION, version))
# initialize internals
self.match_evaluator = MatchEvaluatorUtil(self)
def test_RetryableWriteError_error_label(self):
listener = OvertCommandListener()
client = rs_or_single_client(
retryWrites=True, event_listeners=[listener])
# Ensure collection exists.
client.pymongo_test.testcoll.insert_one({})
with self.fail_point(self.fail_insert):
with self.assertRaises(WriteConcernError) as cm:
client.pymongo_test.testcoll.insert_one({})
self.assertTrue(cm.exception.has_error_label(
'RetryableWriteError'))
if client_context.version >= Version(4, 4):
# In MongoDB 4.4+ we rely on the server returning the error label.
self.assertIn(
'RetryableWriteError',
listener.results['succeeded'][-1].reply['errorLabels'])
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.rs_client = None
self.cmd_line = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.is_rs = False
self.has_ipv6 = False
try:
client = pymongo.MongoClient(host,
port,
serverSelectionTimeoutMS=100)
client.admin.command('ismaster') # Can we connect?
# If so, then reset client to defaults.
self.client = pymongo.MongoClient(host, port)
except pymongo.errors.ConnectionFailure:
self.client = self.rs_or_standalone_client = None
else:
self.connected = True
self.ismaster = self.client.admin.command('ismaster')
self.w = len(self.ismaster.get("hosts", [])) or 1
self.nodes = set([(host, port)])
self.replica_set_name = self.ismaster.get('setName', '')
self.rs_client = None
self.version = Version.from_client(self.client)
if self.replica_set_name:
self.is_rs = True
self.rs_client = pymongo.MongoClient(
pair, replicaSet=self.replica_set_name)
nodes = [
partition_node(node.lower())
for node in self.ismaster.get('hosts', [])
]
nodes.extend([
partition_node(node.lower())
for node in self.ismaster.get('passives', [])
])
nodes.extend([
partition_node(node.lower())
for node in self.ismaster.get('arbiters', [])
])
self.nodes = set(nodes)
self.rs_or_standalone_client = self.rs_client or self.client
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
self.user_provided = self._check_user_provided()
if not self.user_provided:
roles = {}
if self.version.at_least(2, 5, 3, -1):
roles = {'roles': ['root']}
self.client.admin.add_user(db_user, db_pwd, **roles)
self.client.admin.authenticate(db_user, db_pwd)
if self.rs_client:
self.rs_client.admin.authenticate(db_user, db_pwd)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
class ClientContext(object):
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.rs_client = None
self.cmd_line = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.is_rs = False
self.has_ipv6 = False
try:
client = pymongo.MongoClient(host, port,
serverSelectionTimeoutMS=100)
client.admin.command('ismaster') # Can we connect?
# If so, then reset client to defaults.
self.client = pymongo.MongoClient(host, port)
except pymongo.errors.ConnectionFailure:
self.client = None
else:
self.connected = True
self.ismaster = self.client.admin.command('ismaster')
self.w = len(self.ismaster.get("hosts", [])) or 1
self.nodes = set([(host, port)])
self.replica_set_name = self.ismaster.get('setName', '')
self.rs_client = None
self.version = Version.from_client(self.client)
if self.replica_set_name:
self.is_rs = True
self.rs_client = pymongo.MongoClient(
pair, replicaSet=self.replica_set_name)
nodes = [partition_node(node)
for node in self.ismaster.get('hosts', [])]
nodes.extend([partition_node(node)
for node in self.ismaster.get('passives', [])])
nodes.extend([partition_node(node)
for node in self.ismaster.get('arbiters', [])])
self.nodes = set(nodes)
self.rs_or_standalone_client = self.rs_client or self.client
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
self.user_provided = self._check_user_provided()
if not self.user_provided:
roles = {}
if self.version.at_least(2, 5, 3, -1):
roles = {'roles': ['root']}
self.client.admin.add_user(db_user, db_pwd, **roles)
self.client.admin.authenticate(db_user, db_pwd)
if self.rs_client:
self.rs_client.admin.authenticate(db_user, db_pwd)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
def _check_user_provided(self):
try:
self.client.admin.authenticate(db_user, db_pwd)
return True
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 18 or 'auth fails' in msg:
# Auth failed.
return False
else:
raise
def _server_started_with_auth(self):
# MongoDB >= 2.0
if 'parsed' in self.cmd_line:
parsed = self.cmd_line['parsed']
# MongoDB >= 2.6
if 'security' in parsed:
security = parsed['security']
# >= rc3
if 'authorization' in security:
return security['authorization'] == 'enabled'
# < rc3
return (security.get('auth', False) or
bool(security.get('keyFile')))
return parsed.get('auth', False) or bool(parsed.get('keyFile'))
# Legacy
argv = self.cmd_line['argv']
return '--auth' in argv or '--keyFile' in argv
def _server_started_with_ipv6(self):
if not socket.has_ipv6:
return False
if 'parsed' in self.cmd_line:
if not self.cmd_line['parsed'].get('net', {}).get('ipv6'):
return False
else:
if '--ipv6' not in self.cmd_line['argv']:
return False
# The server was started with --ipv6. Is there an IPv6 route to it?
try:
for info in socket.getaddrinfo(host, port):
if info[0] == socket.AF_INET6:
return True
except socket.error:
pass
return False
def _require(self, condition, msg, func=None):
def make_wrapper(f):
@wraps(f)
def wrap(*args, **kwargs):
# Always raise SkipTest if we can't connect to MongoDB
if not self.connected:
raise SkipTest("Cannot connect to MongoDB on %s" % pair)
if condition:
return f(*args, **kwargs)
raise SkipTest(msg)
return wrap
if func is None:
def decorate(f):
return make_wrapper(f)
return decorate
return make_wrapper(func)
def require_connection(self, func):
"""Run a test only if we can connect to MongoDB."""
return self._require(self.connected,
"Cannot connect to MongoDB on %s" % pair,
func=func)
def require_version_min(self, *ver):
"""Run a test only if the server version is at least ``version``."""
other_version = Version(*ver)
return self._require(self.version >= other_version,
"Server version must be at least %s"
% str(other_version))
def require_version_max(self, *ver):
"""Run a test only if the server version is at most ``version``."""
other_version = Version(*ver)
return self._require(self.version <= other_version,
"Server version must be at most %s"
% str(other_version))
def require_auth(self, func):
"""Run a test only if the server is running with auth enabled."""
return self.check_auth_with_sharding(
self._require(self.auth_enabled,
"Authentication is not enabled on the server",
func=func))
def require_no_auth(self, func):
"""Run a test only if the server is running without auth enabled."""
return self._require(not self.auth_enabled,
"Authentication must not be enabled on the server",
func=func)
def require_replica_set(self, func):
"""Run a test only if the client is connected to a replica set."""
return self._require(self.is_rs,
"Not connected to a replica set",
func=func)
def require_no_replica_set(self, func):
"""Run a test if the client is *not* connected to a replica set."""
return self._require(
not self.is_rs,
"Connected to a replica set, not a standalone mongod",
func=func)
def require_ipv6(self, func):
"""Run a test only if the client can connect to a server via IPv6."""
return self._require(self.has_ipv6,
"No IPv6",
func=func)
def require_no_mongos(self, func):
"""Run a test only if the client is not connected to a mongos."""
return self._require(not self.is_mongos,
"Must be connected to a mongod, not a mongos",
func=func)
def require_mongos(self, func):
"""Run a test only if the client is connected to a mongos."""
return self._require(self.is_mongos,
"Must be connected to a mongos",
func=func)
def check_auth_with_sharding(self, func):
"""Skip a test when connected to mongos < 2.0 and running with auth."""
condition = not (self.auth_enabled and
self.is_mongos and self.version < (2,))
return self._require(condition,
"Auth with sharding requires MongoDB >= 2.0.0",
func=func)
def require_test_commands(self, func):
"""Run a test only if the server has test commands enabled."""
return self._require(self.test_commands_enabled,
"Test commands must be enabled",
func=func)
def setup_version(self):
"""Set self.version to the server's version."""
self.version = Version.from_client(self.sync_cx)
class ClientContext(object):
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.rs_client = None
self.cmd_line = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.is_rs = False
self.has_ipv6 = False
try:
client = pymongo.MongoClient(host,
port,
serverSelectionTimeoutMS=100)
client.admin.command('ismaster') # Can we connect?
# If so, then reset client to defaults.
self.client = pymongo.MongoClient(host, port)
except pymongo.errors.ConnectionFailure:
self.client = self.rs_or_standalone_client = None
else:
self.connected = True
self.ismaster = self.client.admin.command('ismaster')
self.w = len(self.ismaster.get("hosts", [])) or 1
self.nodes = set([(host, port)])
self.replica_set_name = self.ismaster.get('setName', '')
self.rs_client = None
self.version = Version.from_client(self.client)
if self.replica_set_name:
self.is_rs = True
self.rs_client = pymongo.MongoClient(
pair, replicaSet=self.replica_set_name)
nodes = [
partition_node(node.lower())
for node in self.ismaster.get('hosts', [])
]
nodes.extend([
partition_node(node.lower())
for node in self.ismaster.get('passives', [])
])
nodes.extend([
partition_node(node.lower())
for node in self.ismaster.get('arbiters', [])
])
self.nodes = set(nodes)
self.rs_or_standalone_client = self.rs_client or self.client
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
self.user_provided = self._check_user_provided()
if not self.user_provided:
roles = {}
if self.version.at_least(2, 5, 3, -1):
roles = {'roles': ['root']}
self.client.admin.add_user(db_user, db_pwd, **roles)
self.client.admin.authenticate(db_user, db_pwd)
if self.rs_client:
self.rs_client.admin.authenticate(db_user, db_pwd)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
def _check_user_provided(self):
try:
self.client.admin.authenticate(db_user, db_pwd)
return True
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 18 or 'auth fails' in msg:
# Auth failed.
return False
else:
raise
def _server_started_with_auth(self):
# MongoDB >= 2.0
if 'parsed' in self.cmd_line:
parsed = self.cmd_line['parsed']
# MongoDB >= 2.6
if 'security' in parsed:
security = parsed['security']
# >= rc3
if 'authorization' in security:
return security['authorization'] == 'enabled'
# < rc3
return (security.get('auth', False)
or bool(security.get('keyFile')))
return parsed.get('auth', False) or bool(parsed.get('keyFile'))
# Legacy
argv = self.cmd_line['argv']
return '--auth' in argv or '--keyFile' in argv
def _server_started_with_ipv6(self):
if not socket.has_ipv6:
return False
if 'parsed' in self.cmd_line:
if not self.cmd_line['parsed'].get('net', {}).get('ipv6'):
return False
else:
if '--ipv6' not in self.cmd_line['argv']:
return False
# The server was started with --ipv6. Is there an IPv6 route to it?
try:
for info in socket.getaddrinfo(host, port):
if info[0] == socket.AF_INET6:
return True
except socket.error:
pass
return False
def _require(self, condition, msg, func=None):
def make_wrapper(f):
@wraps(f)
def wrap(*args, **kwargs):
# Always raise SkipTest if we can't connect to MongoDB
if not self.connected:
raise SkipTest("Cannot connect to MongoDB on %s" % pair)
if condition:
return f(*args, **kwargs)
raise SkipTest(msg)
return wrap
if func is None:
def decorate(f):
return make_wrapper(f)
return decorate
return make_wrapper(func)
def require_connection(self, func):
"""Run a test only if we can connect to MongoDB."""
return self._require(self.connected,
"Cannot connect to MongoDB on %s" % pair,
func=func)
def require_version_min(self, *ver):
"""Run a test only if the server version is at least ``version``."""
other_version = Version(*ver)
return self._require(
self.version >= other_version,
"Server version must be at least %s" % str(other_version))
def require_version_max(self, *ver):
"""Run a test only if the server version is at most ``version``."""
other_version = Version(*ver)
return self._require(
self.version <= other_version,
"Server version must be at most %s" % str(other_version))
def require_auth(self, func):
"""Run a test only if the server is running with auth enabled."""
return self.check_auth_with_sharding(
self._require(self.auth_enabled,
"Authentication is not enabled on the server",
func=func))
def require_no_auth(self, func):
"""Run a test only if the server is running without auth enabled."""
return self._require(
not self.auth_enabled,
"Authentication must not be enabled on the server",
func=func)
def require_replica_set(self, func):
"""Run a test only if the client is connected to a replica set."""
return self._require(self.is_rs,
"Not connected to a replica set",
func=func)
def require_no_replica_set(self, func):
"""Run a test if the client is *not* connected to a replica set."""
return self._require(
not self.is_rs,
"Connected to a replica set, not a standalone mongod",
func=func)
def require_ipv6(self, func):
"""Run a test only if the client can connect to a server via IPv6."""
return self._require(self.has_ipv6, "No IPv6", func=func)
def require_no_mongos(self, func):
"""Run a test only if the client is not connected to a mongos."""
return self._require(not self.is_mongos,
"Must be connected to a mongod, not a mongos",
func=func)
def require_mongos(self, func):
"""Run a test only if the client is connected to a mongos."""
return self._require(self.is_mongos,
"Must be connected to a mongos",
func=func)
def check_auth_with_sharding(self, func):
"""Skip a test when connected to mongos < 2.0 and running with auth."""
condition = not (self.auth_enabled and self.is_mongos
and self.version < (2, ))
return self._require(condition,
"Auth with sharding requires MongoDB >= 2.0.0",
func=func)
def require_test_commands(self, func):
"""Run a test only if the server has test commands enabled."""
return self._require(self.test_commands_enabled,
"Test commands must be enabled",
func=func)
class ClientContext(object):
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.cmd_line = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.is_rs = False
self.has_ipv6 = False
self.ssl = False
self.ssl_cert_none = False
self.ssl_certfile = False
self.server_is_resolvable = is_server_resolvable()
self.ssl_client_options = {}
self.client = _connect(host, port)
if HAVE_SSL and not self.client:
# Is MongoDB configured for SSL?
self.client = _connect(host, port, **_SSL_OPTIONS)
if self.client:
self.ssl = True
self.ssl_client_options = _SSL_OPTIONS
self.ssl_certfile = True
if _SSL_OPTIONS.get('ssl_cert_reqs') == ssl.CERT_NONE:
self.ssl_cert_none = True
if self.client:
self.connected = True
ismaster = self.client.admin.command('ismaster')
if 'setName' in ismaster:
self.replica_set_name = ismaster['setName']
self.is_rs = True
# It doesn't matter which member we use as the seed here.
self.client = pymongo.MongoClient(
host,
port,
replicaSet=self.replica_set_name,
**self.ssl_client_options)
# Get the authoritative ismaster result from the primary.
self.ismaster = self.client.admin.command('ismaster')
nodes = [partition_node(node.lower())
for node in self.ismaster.get('hosts', [])]
nodes.extend([partition_node(node.lower())
for node in self.ismaster.get('passives', [])])
nodes.extend([partition_node(node.lower())
for node in self.ismaster.get('arbiters', [])])
self.nodes = set(nodes)
else:
self.ismaster = ismaster
self.nodes = set([(host, port)])
self.w = len(self.ismaster.get("hosts", [])) or 1
self.version = Version.from_client(self.client)
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
self.user_provided = self._check_user_provided()
if not self.user_provided:
roles = {}
if self.version.at_least(2, 5, 3, -1):
roles = {'roles': ['root']}
self.client.admin.add_user(db_user, db_pwd, **roles)
self.client.admin.authenticate(db_user, db_pwd)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
else:
params = self.cmd_line['parsed'].get('setParameter', {})
if params.get('enableTestCommands') == '1':
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
@property
def host(self):
if self.is_rs:
primary = self.client.primary
return primary[0] if primary is not None else host
return host
@property
def port(self):
if self.is_rs:
primary = self.client.primary
return primary[1] if primary is not None else port
return port
@property
def pair(self):
return "%s:%d" % (self.host, self.port)
@property
def has_secondaries(self):
if not self.client:
return False
return bool(len(self.client.secondaries))
def _check_user_provided(self):
try:
self.client.admin.authenticate(db_user, db_pwd)
return True
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 18 or 'auth fails' in msg:
# Auth failed.
return False
else:
raise
def _server_started_with_auth(self):
# MongoDB >= 2.0
if 'parsed' in self.cmd_line:
parsed = self.cmd_line['parsed']
# MongoDB >= 2.6
if 'security' in parsed:
security = parsed['security']
# >= rc3
if 'authorization' in security:
return security['authorization'] == 'enabled'
# < rc3
return (security.get('auth', False) or
bool(security.get('keyFile')))
return parsed.get('auth', False) or bool(parsed.get('keyFile'))
# Legacy
argv = self.cmd_line['argv']
return '--auth' in argv or '--keyFile' in argv
def _server_started_with_ipv6(self):
if not socket.has_ipv6:
return False
if 'parsed' in self.cmd_line:
if not self.cmd_line['parsed'].get('net', {}).get('ipv6'):
return False
else:
if '--ipv6' not in self.cmd_line['argv']:
return False
# The server was started with --ipv6. Is there an IPv6 route to it?
try:
for info in socket.getaddrinfo(self.host, self.port):
if info[0] == socket.AF_INET6:
return True
except socket.error:
pass
return False
def _require(self, condition, msg, func=None):
def make_wrapper(f):
@wraps(f)
def wrap(*args, **kwargs):
# Always raise SkipTest if we can't connect to MongoDB
if not self.connected:
raise SkipTest(
"Cannot connect to MongoDB on %s" % (self.pair,))
if condition:
return f(*args, **kwargs)
raise SkipTest(msg)
return wrap
if func is None:
def decorate(f):
return make_wrapper(f)
return decorate
return make_wrapper(func)
def require_connection(self, func):
"""Run a test only if we can connect to MongoDB."""
return self._require(
self.connected,
"Cannot connect to MongoDB on %s" % (self.pair,),
func=func)
def require_version_min(self, *ver):
"""Run a test only if the server version is at least ``version``."""
other_version = Version(*ver)
return self._require(self.version >= other_version,
"Server version must be at least %s"
% str(other_version))
def require_version_max(self, *ver):
"""Run a test only if the server version is at most ``version``."""
other_version = Version(*ver)
return self._require(self.version <= other_version,
"Server version must be at most %s"
% str(other_version))
def require_auth(self, func):
"""Run a test only if the server is running with auth enabled."""
return self.check_auth_with_sharding(
self._require(self.auth_enabled,
"Authentication is not enabled on the server",
func=func))
def require_no_auth(self, func):
"""Run a test only if the server is running without auth enabled."""
return self._require(not self.auth_enabled,
"Authentication must not be enabled on the server",
func=func)
def require_replica_set(self, func):
"""Run a test only if the client is connected to a replica set."""
return self._require(self.is_rs,
"Not connected to a replica set",
func=func)
def require_secondaries_count(self, count):
"""Run a test only if the client is connected to a replica set that has
`count` secondaries.
"""
sec_count = 0 if not self.client else len(self.client.secondaries)
return self._require(sec_count >= count,
"Need %d secondaries, %d available"
% (count, sec_count))
def require_no_replica_set(self, func):
"""Run a test if the client is *not* connected to a replica set."""
return self._require(
not self.is_rs,
"Connected to a replica set, not a standalone mongod",
func=func)
def require_ipv6(self, func):
"""Run a test only if the client can connect to a server via IPv6."""
return self._require(self.has_ipv6,
"No IPv6",
func=func)
def require_no_mongos(self, func):
"""Run a test only if the client is not connected to a mongos."""
return self._require(not self.is_mongos,
"Must be connected to a mongod, not a mongos",
func=func)
def require_mongos(self, func):
"""Run a test only if the client is connected to a mongos."""
return self._require(self.is_mongos,
"Must be connected to a mongos",
func=func)
def check_auth_with_sharding(self, func):
"""Skip a test when connected to mongos < 2.0 and running with auth."""
condition = not (self.auth_enabled and
self.is_mongos and self.version < (2,))
return self._require(condition,
"Auth with sharding requires MongoDB >= 2.0.0",
func=func)
def require_test_commands(self, func):
"""Run a test only if the server has test commands enabled."""
return self._require(self.test_commands_enabled,
"Test commands must be enabled",
func=func)
def require_ssl(self, func):
"""Run a test only if the client can connect over SSL."""
return self._require(self.ssl,
"Must be able to connect via SSL",
func=func)
def require_no_ssl(self, func):
"""Run a test only if the client can connect over SSL."""
return self._require(not self.ssl,
"Must be able to connect without SSL",
func=func)
def require_ssl_cert_none(self, func):
"""Run a test only if the client can connect with ssl.CERT_NONE."""
return self._require(self.ssl_cert_none,
"Must be able to connect with ssl.CERT_NONE",
func=func)
def require_ssl_certfile(self, func):
"""Run a test only if the client can connect with ssl_certfile."""
return self._require(self.ssl_certfile,
"Must be able to connect with ssl_certfile",
func=func)
def require_server_resolvable(self, func):
"""Run a test only if the hostname 'server' is resolvable."""
return self._require(self.server_is_resolvable,
"No hosts entry for 'server'. Cannot validate "
"hostname in the certificate",
func=func)
def _init_client(self):
self.client = self._connect(host, port)
if self.client is not None:
# Return early when connected to dataLake as mongohoused does not
# support the getCmdLineOpts command and is tested without TLS.
build_info = self.client.admin.command('buildInfo')
if 'dataLake' in build_info:
self.is_data_lake = True
self.auth_enabled = True
self.client = self._connect(host,
port,
username=db_user,
password=db_pwd)
self.connected = True
return
if HAVE_SSL and not self.client:
# Is MongoDB configured for SSL?
self.client = self._connect(host, port, **TLS_OPTIONS)
if self.client:
self.tls = True
self.default_client_options.update(TLS_OPTIONS)
self.ssl_certfile = True
if self.client:
self.connected = True
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
if not self._check_user_provided():
_create_user(self.client.admin, db_user, db_pwd)
self.client = self._connect(host,
port,
username=db_user,
password=db_pwd,
replicaSet=self.replica_set_name,
**self.default_client_options)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
self.server_status = self.client.admin.command('serverStatus')
if self.storage_engine == "mmapv1":
# MMAPv1 does not support retryWrites=True.
self.default_client_options['retryWrites'] = False
ismaster = self.ismaster
self.sessions_enabled = 'logicalSessionTimeoutMinutes' in ismaster
if 'setName' in ismaster:
self.replica_set_name = str(ismaster['setName'])
self.is_rs = True
if self.auth_enabled:
# It doesn't matter which member we use as the seed here.
self.client = pymongo.MongoClient(
host,
port,
username=db_user,
password=db_pwd,
replicaSet=self.replica_set_name,
**self.default_client_options)
else:
self.client = pymongo.MongoClient(
host,
port,
replicaSet=self.replica_set_name,
**self.default_client_options)
# Get the authoritative ismaster result from the primary.
ismaster = self.ismaster
nodes = [
partition_node(node.lower())
for node in ismaster.get('hosts', [])
]
nodes.extend([
partition_node(node.lower())
for node in ismaster.get('passives', [])
])
nodes.extend([
partition_node(node.lower())
for node in ismaster.get('arbiters', [])
])
self.nodes = set(nodes)
else:
self.nodes = set([(host, port)])
self.w = len(ismaster.get("hosts", [])) or 1
self.version = Version.from_client(self.client)
self.server_parameters = self.client.admin.command(
'getParameter', '*')
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
else:
params = self.cmd_line['parsed'].get('setParameter', {})
if params.get('enableTestCommands') == '1':
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
if self.is_mongos:
# Check for another mongos on the next port.
address = self.client.address
next_address = address[0], address[1] + 1
self.mongoses.append(address)
mongos_client = self._connect(*next_address,
**self.default_client_options)
if mongos_client:
ismaster = mongos_client.admin.command('ismaster')
if ismaster.get('msg') == 'isdbgrid':
self.mongoses.append(next_address)
def require_version_min(self, *ver):
"""Run a test only if the server version is at least ``version``."""
other_version = Version(*ver)
return self._require(
self.version >= other_version,
"Server version must be at least %s" % str(other_version))
def __init__(self):
"""Create a client and grab essential information from the server."""
# Seed host. This may be updated further down.
self.host, self.port = host, port
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.rs_client = None
self.cmd_line = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.is_rs = False
self.has_ipv6 = False
self.ssl_cert_none = False
self.ssl_certfile = False
self.server_is_resolvable = is_server_resolvable()
self.client = self.rs_or_standalone_client = None
def connect(**kwargs):
try:
client = pymongo.MongoClient(self.host,
self.port,
serverSelectionTimeoutMS=100,
**kwargs)
client.admin.command('ismaster') # Can we connect?
# If connected, then return client with default timeout
return pymongo.MongoClient(self.host, self.port, **kwargs)
except pymongo.errors.ConnectionFailure:
return None
self.client = connect()
if HAVE_SSL and not self.client:
# Is MongoDB configured for SSL?
self.client = connect(ssl=True, ssl_cert_reqs=ssl.CERT_NONE)
if self.client:
self.ssl_cert_none = True
# Can client connect with certfile?
client = connect(
ssl=True,
ssl_cert_reqs=ssl.CERT_NONE,
ssl_certfile=CLIENT_PEM,
)
if client:
self.ssl_certfile = True
self.client = client
if self.client:
self.connected = True
self.ismaster = self.client.admin.command('ismaster')
self.w = len(self.ismaster.get("hosts", [])) or 1
self.nodes = set([(self.host, self.port)])
self.replica_set_name = self.ismaster.get('setName', '')
self.rs_client = None
self.version = Version.from_client(self.client)
if self.replica_set_name:
self.is_rs = True
self.rs_client = pymongo.MongoClient(
self.ismaster['primary'], replicaSet=self.replica_set_name)
# Force connection
self.rs_client.admin.command('ismaster')
self.host, self.port = self.rs_client.primary
self.client = connect()
nodes = [
partition_node(node.lower())
for node in self.ismaster.get('hosts', [])
]
nodes.extend([
partition_node(node.lower())
for node in self.ismaster.get('passives', [])
])
nodes.extend([
partition_node(node.lower())
for node in self.ismaster.get('arbiters', [])
])
self.nodes = set(nodes)
self.rs_or_standalone_client = self.rs_client or self.client
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
self.user_provided = self._check_user_provided()
if not self.user_provided:
roles = {}
if self.version.at_least(2, 5, 3, -1):
roles = {'roles': ['root']}
self.client.admin.add_user(db_user, db_pwd, **roles)
self.client.admin.authenticate(db_user, db_pwd)
if self.rs_client:
self.rs_client.admin.authenticate(db_user, db_pwd)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
# Do this after we connect so we know who the primary is.
self.pair = "%s:%d" % (self.host, self.port)
def test_mongodb_x509_auth(self):
# Expects the server to be running with the server.pem and ca.pem
# as well as --auth
#
# --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem
# --sslCAFile=/path/to/pymongo/test/certificates/ca.pem
# --auth
if not CERT_SSL:
raise SkipTest("No mongod available over SSL with certs")
if not Version.from_client(ssl_client).at_least(2, 5, 3, -1):
raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer")
if not server_started_with_auth(ssl_client):
raise SkipTest('Authentication is not enabled on server')
self.addCleanup(ssl_client['$external'].logout)
self.addCleanup(remove_all_users, ssl_client['$external'])
self.addCleanup(remove_all_users, ssl_client.admin)
ssl_client.admin.add_user('admin', 'pass')
ssl_client.admin.authenticate('admin', 'pass')
# Give admin all necessary privileges.
ssl_client['$external'].add_user(MONGODB_X509_USERNAME,
roles=[{
'role': 'readWriteAnyDatabase',
'db': 'admin'
}, {
'role': 'userAdminAnyDatabase',
'db': 'admin'
}])
ssl_client.admin.logout()
coll = ssl_client.pymongo_test.test
self.assertRaises(OperationFailure, coll.count)
self.assertTrue(
ssl_client.admin.authenticate(MONGODB_X509_USERNAME,
mechanism='MONGODB-X509'))
coll.drop()
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' %
(quote_plus(MONGODB_X509_USERNAME), host, port))
# SSL options aren't supported in the URI...
self.assertTrue(MongoClient(uri, ssl=True, ssl_certfile=CLIENT_PEM))
# Should require a username
uri = ('mongodb://%s:%d/?authMechanism=MONGODB-X509' % (host, port))
client_bad = MongoClient(uri,
ssl=True,
ssl_cert_reqs="CERT_NONE",
ssl_certfile=CLIENT_PEM)
self.assertRaises(OperationFailure,
client_bad.pymongo_test.test.delete_one, {})
# Auth should fail if username and certificate do not match
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' % (quote_plus("not the username"), host, port))
bad_client = MongoClient(uri,
ssl=True,
ssl_cert_reqs="CERT_NONE",
ssl_certfile=CLIENT_PEM)
with self.assertRaises(OperationFailure):
bad_client.pymongo_test.test.find_one()
self.assertRaises(OperationFailure,
ssl_client.admin.authenticate,
"not the username",
mechanism="MONGODB-X509")
# Invalid certificate (using CA certificate as client certificate)
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' %
(quote_plus(MONGODB_X509_USERNAME), host, port))
try:
connected(
MongoClient(uri,
ssl=True,
ssl_cert_reqs="CERT_NONE",
ssl_certfile=CA_PEM,
serverSelectionTimeoutMS=100))
except OperationFailure:
pass
else:
self.fail("Invalid certificate accepted.")
def __init__(self):
"""Create a client and grab essential information from the server."""
# Seed host. This may be updated further down.
self.host, self.port = host, port
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.cmd_line = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.is_rs = False
self.has_ipv6 = False
self.ssl = False
self.ssl_cert_none = False
self.ssl_certfile = False
self.server_is_resolvable = is_server_resolvable()
self.ssl_client_options = {}
self.client = _connect(self.host, self.port)
if HAVE_SSL and not self.client:
# Is MongoDB configured for SSL?
self.client = _connect(self.host, self.port, **_SSL_OPTIONS)
if self.client:
self.ssl = True
self.ssl_client_options = _SSL_OPTIONS
self.ssl_certfile = True
if _SSL_OPTIONS.get('ssl_cert_reqs') == ssl.CERT_NONE:
self.ssl_cert_none = True
if self.client:
self.connected = True
self.ismaster = self.client.admin.command('ismaster')
self.w = len(self.ismaster.get("hosts", [])) or 1
self.nodes = set([(self.host, self.port)])
self.replica_set_name = self.ismaster.get('setName', '')
self.version = Version.from_client(self.client)
if self.replica_set_name:
self.is_rs = True
self.client = pymongo.MongoClient(
self.ismaster['primary'],
replicaSet=self.replica_set_name,
**self.ssl_client_options)
# Force connection
self.client.admin.command('ismaster')
self.host, self.port = self.client.primary
nodes = [partition_node(node.lower())
for node in self.ismaster.get('hosts', [])]
nodes.extend([partition_node(node.lower())
for node in self.ismaster.get('passives', [])])
nodes.extend([partition_node(node.lower())
for node in self.ismaster.get('arbiters', [])])
self.nodes = set(nodes)
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
self.user_provided = self._check_user_provided()
if not self.user_provided:
roles = {}
if self.version.at_least(2, 5, 3, -1):
roles = {'roles': ['root']}
self.client.admin.add_user(db_user, db_pwd, **roles)
self.client.admin.authenticate(db_user, db_pwd)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
else:
params = self.cmd_line['parsed'].get('setParameter', {})
if params.get('enableTestCommands') == '1':
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
class UnifiedSpecTestMixinV1(IntegrationTest):
"""Mixin class to run test cases from test specification files.
Assumes that tests conform to the `unified test format
<https://github.com/mongodb/specifications/blob/master/source/unified-test-format/unified-test-format.rst>`_.
Specification of the test suite being currently run is available as
a class attribute ``TEST_SPEC``.
"""
SCHEMA_VERSION = Version.from_string('1.1')
@staticmethod
def should_run_on(run_on_spec):
if not run_on_spec:
# Always run these tests.
return True
for req in run_on_spec:
if is_run_on_requirement_satisfied(req):
return True
return False
def insert_initial_data(self, initial_data):
for collection_data in initial_data:
coll_name = collection_data['collectionName']
db_name = collection_data['databaseName']
documents = collection_data['documents']
coll = self.client.get_database(db_name).get_collection(
coll_name, write_concern=WriteConcern(w="majority"))
coll.drop()
if len(documents) > 0:
coll.insert_many(documents)
else:
# ensure collection exists
result = coll.insert_one({})
coll.delete_one({'_id': result.inserted_id})
@classmethod
def setUpClass(cls):
# super call creates internal client cls.client
super(UnifiedSpecTestMixinV1, cls).setUpClass()
# process file-level runOnRequirements
run_on_spec = cls.TEST_SPEC.get('runOnRequirements', [])
if not cls.should_run_on(run_on_spec):
raise unittest.SkipTest('%s runOnRequirements not satisfied' %
(cls.__name__, ))
# add any special-casing for skipping tests here
if client_context.storage_engine == 'mmapv1':
if 'retryable-writes' in cls.TEST_SPEC['description']:
raise unittest.SkipTest(
"MMAPv1 does not support retryWrites=True")
@classmethod
def tearDownClass(cls):
super(UnifiedSpecTestMixinV1, cls).tearDownClass()
cls.client.close()
def setUp(self):
super(UnifiedSpecTestMixinV1, self).setUp()
# process schemaVersion
# note: we check major schema version during class generation
# note: we do this here because we cannot run assertions in setUpClass
version = Version.from_string(self.TEST_SPEC['schemaVersion'])
self.assertLessEqual(
version, self.SCHEMA_VERSION,
'expected schema version %s or lower, got %s' %
(self.SCHEMA_VERSION, version))
# initialize internals
self.match_evaluator = MatchEvaluatorUtil(self)
def maybe_skip_test(self, spec):
# add any special-casing for skipping tests here
if client_context.storage_engine == 'mmapv1':
if 'Dirty explicit session is discarded' in spec['description']:
raise unittest.SkipTest(
"MMAPv1 does not support retryWrites=True")
def process_error(self, exception, spec):
is_error = spec.get('isError')
is_client_error = spec.get('isClientError')
error_contains = spec.get('errorContains')
error_code = spec.get('errorCode')
error_code_name = spec.get('errorCodeName')
error_labels_contain = spec.get('errorLabelsContain')
error_labels_omit = spec.get('errorLabelsOmit')
expect_result = spec.get('expectResult')
if is_error:
# already satisfied because exception was raised
pass
if is_client_error:
self.assertNotIsInstance(exception, PyMongoError)
if error_contains:
if isinstance(exception, BulkWriteError):
errmsg = str(exception.details).lower()
else:
errmsg = str(exception).lower()
self.assertIn(error_contains.lower(), errmsg)
if error_code:
self.assertEqual(error_code, exception.details.get('code'))
if error_code_name:
self.assertEqual(error_code_name,
exception.details.get('codeName'))
if error_labels_contain:
labels = [
err_label for err_label in error_labels_contain
if exception.has_error_label(err_label)
]
self.assertEqual(labels, error_labels_contain)
if error_labels_omit:
for err_label in error_labels_omit:
if exception.has_error_label(err_label):
self.fail("Exception '%s' unexpectedly had label '%s'" %
(exception, err_label))
if expect_result:
if isinstance(exception, BulkWriteError):
result = parse_bulk_write_error_result(exception)
self.match_evaluator.match_result(expect_result, result)
else:
self.fail("expectResult can only be specified with %s "
"exceptions" % (BulkWriteError, ))
def __raise_if_unsupported(self, opname, target, *target_types):
if not isinstance(target, target_types):
self.fail('Operation %s not supported for entity '
'of type %s' % (opname, type(target)))
def __entityOperation_createChangeStream(self, target, *args, **kwargs):
if client_context.storage_engine == 'mmapv1':
self.skipTest("MMAPv1 does not support change streams")
self.__raise_if_unsupported('createChangeStream', target, MongoClient,
Database, Collection)
return target.watch(*args, **kwargs)
def _clientOperation_createChangeStream(self, target, *args, **kwargs):
return self.__entityOperation_createChangeStream(
target, *args, **kwargs)
def _databaseOperation_createChangeStream(self, target, *args, **kwargs):
return self.__entityOperation_createChangeStream(
target, *args, **kwargs)
def _collectionOperation_createChangeStream(self, target, *args, **kwargs):
return self.__entityOperation_createChangeStream(
target, *args, **kwargs)
def _databaseOperation_runCommand(self, target, **kwargs):
self.__raise_if_unsupported('runCommand', target, Database)
# Ensure the first key is the command name.
ordered_command = SON([(kwargs.pop('command_name'), 1)])
ordered_command.update(kwargs['command'])
kwargs['command'] = ordered_command
return target.command(**kwargs)
def __entityOperation_aggregate(self, target, *args, **kwargs):
self.__raise_if_unsupported('aggregate', target, Database, Collection)
return list(target.aggregate(*args, **kwargs))
def _databaseOperation_aggregate(self, target, *args, **kwargs):
return self.__entityOperation_aggregate(target, *args, **kwargs)
def _collectionOperation_aggregate(self, target, *args, **kwargs):
return self.__entityOperation_aggregate(target, *args, **kwargs)
def _collectionOperation_bulkWrite(self, target, *args, **kwargs):
self.__raise_if_unsupported('bulkWrite', target, Collection)
write_result = target.bulk_write(*args, **kwargs)
return parse_bulk_write_result(write_result)
def _collectionOperation_find(self, target, *args, **kwargs):
self.__raise_if_unsupported('find', target, Collection)
find_cursor = target.find(*args, **kwargs)
return list(find_cursor)
def _collectionOperation_findOneAndReplace(self, target, *args, **kwargs):
self.__raise_if_unsupported('findOneAndReplace', target, Collection)
return target.find_one_and_replace(*args, **kwargs)
def _collectionOperation_findOneAndUpdate(self, target, *args, **kwargs):
self.__raise_if_unsupported('findOneAndReplace', target, Collection)
return target.find_one_and_update(*args, **kwargs)
def _collectionOperation_insertMany(self, target, *args, **kwargs):
self.__raise_if_unsupported('insertMany', target, Collection)
result = target.insert_many(*args, **kwargs)
return {idx: _id for idx, _id in enumerate(result.inserted_ids)}
def _collectionOperation_insertOne(self, target, *args, **kwargs):
self.__raise_if_unsupported('insertOne', target, Collection)
result = target.insert_one(*args, **kwargs)
return {'insertedId': result.inserted_id}
def _sessionOperation_withTransaction(self, target, *args, **kwargs):
if client_context.storage_engine == 'mmapv1':
self.skipTest('MMAPv1 does not support document-level locking')
self.__raise_if_unsupported('withTransaction', target, ClientSession)
return target.with_transaction(*args, **kwargs)
def _sessionOperation_startTransaction(self, target, *args, **kwargs):
if client_context.storage_engine == 'mmapv1':
self.skipTest('MMAPv1 does not support document-level locking')
self.__raise_if_unsupported('startTransaction', target, ClientSession)
return target.start_transaction(*args, **kwargs)
def _changeStreamOperation_iterateUntilDocumentOrError(
self, target, *args, **kwargs):
self.__raise_if_unsupported('iterateUntilDocumentOrError', target,
ChangeStream)
return next(target)
def run_entity_operation(self, spec):
target = self.entity_map[spec['object']]
opname = spec['name']
opargs = spec.get('arguments')
expect_error = spec.get('expectError')
if opargs:
arguments = parse_spec_options(copy.deepcopy(opargs))
prepare_spec_arguments(spec, arguments, camel_to_snake(opname),
self.entity_map, self.run_operations)
else:
arguments = tuple()
if isinstance(target, MongoClient):
method_name = '_clientOperation_%s' % (opname, )
elif isinstance(target, Database):
method_name = '_databaseOperation_%s' % (opname, )
elif isinstance(target, Collection):
method_name = '_collectionOperation_%s' % (opname, )
elif isinstance(target, ChangeStream):
method_name = '_changeStreamOperation_%s' % (opname, )
elif isinstance(target, ClientSession):
method_name = '_sessionOperation_%s' % (opname, )
elif isinstance(target, GridFSBucket):
raise NotImplementedError
else:
method_name = 'doesNotExist'
try:
method = getattr(self, method_name)
except AttributeError:
try:
cmd = getattr(target, camel_to_snake(opname))
except AttributeError:
self.fail('Unsupported operation %s on entity %s' %
(opname, target))
else:
cmd = functools.partial(method, target)
try:
result = cmd(**dict(arguments))
except Exception as exc:
if expect_error:
return self.process_error(exc, expect_error)
raise
if 'expectResult' in spec:
self.match_evaluator.match_result(spec['expectResult'], result)
save_as_entity = spec.get('saveResultAsEntity')
if save_as_entity:
self.entity_map[save_as_entity] = result
def __set_fail_point(self, client, command_args):
if not client_context.test_commands_enabled:
self.skipTest('Test commands must be enabled')
cmd_on = SON([('configureFailPoint', 'failCommand')])
cmd_on.update(command_args)
client.admin.command(cmd_on)
self.addCleanup(client.admin.command,
'configureFailPoint',
cmd_on['configureFailPoint'],
mode='off')
def _testOperation_failPoint(self, spec):
self.__set_fail_point(client=self.entity_map[spec['client']],
command_args=spec['failPoint'])
def _testOperation_targetedFailPoint(self, spec):
session = self.entity_map[spec['session']]
if not session._pinned_address:
self.fail("Cannot use targetedFailPoint operation with unpinned "
"session %s" % (spec['session'], ))
client = single_client('%s:%s' % session._pinned_address)
self.__set_fail_point(client=client, command_args=spec['failPoint'])
self.addCleanup(client.close)
def _testOperation_assertSessionTransactionState(self, spec):
session = self.entity_map[spec['session']]
expected_state = getattr(_TxnState, spec['state'].upper())
self.assertEqual(expected_state, session._transaction.state)
def _testOperation_assertSessionPinned(self, spec):
session = self.entity_map[spec['session']]
self.assertIsNotNone(session._pinned_address)
def _testOperation_assertSessionUnpinned(self, spec):
session = self.entity_map[spec['session']]
self.assertIsNone(session._pinned_address)
def __get_last_two_command_lsids(self, listener):
cmd_started_events = []
for event in reversed(listener.results):
if isinstance(event, CommandStartedEvent):
cmd_started_events.append(event)
if len(cmd_started_events) < 2:
self.fail('Needed 2 CommandStartedEvents to compare lsids, '
'got %s' % (len(cmd_started_events)))
return tuple([e.command['lsid'] for e in cmd_started_events][:2])
def _testOperation_assertDifferentLsidOnLastTwoCommands(self, spec):
listener = self.entity_map.get_listener_for_client(spec['client'])
self.assertNotEqual(*self.__get_last_two_command_lsids(listener))
def _testOperation_assertSameLsidOnLastTwoCommands(self, spec):
listener = self.entity_map.get_listener_for_client(spec['client'])
self.assertEqual(*self.__get_last_two_command_lsids(listener))
def _testOperation_assertSessionDirty(self, spec):
session = self.entity_map[spec['session']]
self.assertTrue(session._server_session.dirty)
def _testOperation_assertSessionNotDirty(self, spec):
session = self.entity_map[spec['session']]
return self.assertFalse(session._server_session.dirty)
def _testOperation_assertCollectionExists(self, spec):
database_name = spec['databaseName']
collection_name = spec['collectionName']
collection_name_list = list(
self.client.get_database(database_name).list_collection_names())
self.assertIn(collection_name, collection_name_list)
def _testOperation_assertCollectionNotExists(self, spec):
database_name = spec['databaseName']
collection_name = spec['collectionName']
collection_name_list = list(
self.client.get_database(database_name).list_collection_names())
self.assertNotIn(collection_name, collection_name_list)
def _testOperation_assertIndexExists(self, spec):
collection = self.client[spec['databaseName']][spec['collectionName']]
index_names = [idx['name'] for idx in collection.list_indexes()]
self.assertIn(spec['indexName'], index_names)
def _testOperation_assertIndexNotExists(self, spec):
collection = self.client[spec['databaseName']][spec['collectionName']]
for index in collection.list_indexes():
self.assertNotEqual(spec['indexName'], index['name'])
def run_special_operation(self, spec):
opname = spec['name']
method_name = '_testOperation_%s' % (opname, )
try:
method = getattr(self, method_name)
except AttributeError:
self.fail('Unsupported special test operation %s' % (opname, ))
else:
method(spec['arguments'])
def run_operations(self, spec):
for op in spec:
target = op['object']
if target != 'testRunner':
self.run_entity_operation(op)
else:
self.run_special_operation(op)
def check_events(self, spec):
for event_spec in spec:
client_name = event_spec['client']
events = event_spec['events']
listener = self.entity_map.get_listener_for_client(client_name)
if len(events) == 0:
self.assertEqual(listener.results, [])
continue
if len(events) > len(listener.results):
self.fail('Expected to see %s events, got %s' %
(len(events), len(listener.results)))
for idx, expected_event in enumerate(events):
self.match_evaluator.match_event(expected_event,
listener.results[idx])
def verify_outcome(self, spec):
for collection_data in spec:
coll_name = collection_data['collectionName']
db_name = collection_data['databaseName']
expected_documents = collection_data['documents']
coll = self.client.get_database(db_name).get_collection(
coll_name,
read_preference=ReadPreference.PRIMARY,
read_concern=ReadConcern(level='local'))
if expected_documents:
sorted_expected_documents = sorted(expected_documents,
key=lambda doc: doc['_id'])
actual_documents = list(
coll.find({}, sort=[('_id', ASCENDING)]))
self.assertListEqual(sorted_expected_documents,
actual_documents)
def run_scenario(self, spec):
# maybe skip test manually
self.maybe_skip_test(spec)
# process test-level runOnRequirements
run_on_spec = spec.get('runOnRequirements', [])
if not self.should_run_on(run_on_spec):
raise unittest.SkipTest('runOnRequirements not satisfied')
# process skipReason
skip_reason = spec.get('skipReason', None)
if skip_reason is not None:
raise unittest.SkipTest('%s' % (skip_reason, ))
# process createEntities
self.entity_map = EntityMapUtil(self)
self.entity_map.create_entities_from_spec(
self.TEST_SPEC.get('createEntities', []))
# process initialData
self.insert_initial_data(self.TEST_SPEC.get('initialData', []))
# process operations
self.run_operations(spec['operations'])
# process expectEvents
self.check_events(spec.get('expectEvents', []))
# process outcome
self.verify_outcome(spec.get('outcome', []))
def _init_client(self):
self.client = self._connect(host, port)
if HAVE_SSL and not self.client:
# Is MongoDB configured for SSL?
self.client = self._connect(host, port, **_SSL_OPTIONS)
if self.client:
self.ssl = True
self.default_client_options.update(_SSL_OPTIONS)
self.ssl_certfile = True
if _SSL_OPTIONS.get('ssl_cert_reqs') == ssl.CERT_NONE:
self.ssl_cert_none = True
if self.client:
self.connected = True
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
if not self._check_user_provided():
_create_user(self.client.admin, db_user, db_pwd)
self.client = self._connect(
host, port, username=db_user, password=db_pwd,
replicaSet=self.replica_set_name,
**self.default_client_options)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
self.server_status = self.client.admin.command('serverStatus')
self.ismaster = ismaster = self.client.admin.command('isMaster')
self.sessions_enabled = 'logicalSessionTimeoutMinutes' in ismaster
if 'setName' in ismaster:
self.replica_set_name = str(ismaster['setName'])
self.is_rs = True
if self.auth_enabled:
# It doesn't matter which member we use as the seed here.
self.client = pymongo.MongoClient(
host,
port,
username=db_user,
password=db_pwd,
replicaSet=self.replica_set_name,
**self.default_client_options)
else:
self.client = pymongo.MongoClient(
host,
port,
replicaSet=self.replica_set_name,
**self.default_client_options)
# Get the authoritative ismaster result from the primary.
self.ismaster = self.client.admin.command('ismaster')
nodes = [partition_node(node.lower())
for node in self.ismaster.get('hosts', [])]
nodes.extend([partition_node(node.lower())
for node in self.ismaster.get('passives', [])])
nodes.extend([partition_node(node.lower())
for node in self.ismaster.get('arbiters', [])])
self.nodes = set(nodes)
else:
self.ismaster = ismaster
self.nodes = set([(host, port)])
self.w = len(self.ismaster.get("hosts", [])) or 1
self.version = Version.from_client(self.client)
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
else:
params = self.cmd_line['parsed'].get('setParameter', {})
if params.get('enableTestCommands') == '1':
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
if self.is_mongos:
# Check for another mongos on the next port.
address = self.client.address
next_address = address[0], address[1] + 1
self.mongoses.append(address)
mongos_client = self._connect(*next_address,
**self.default_client_options)
if mongos_client:
ismaster = mongos_client.admin.command('ismaster')
if ismaster.get('msg') == 'isdbgrid':
self.mongoses.append(next_address)
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.cmd_line = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.is_rs = False
self.has_ipv6 = False
self.ssl = False
self.ssl_cert_none = False
self.ssl_certfile = False
self.server_is_resolvable = is_server_resolvable()
self.ssl_client_options = {}
self.client = _connect(host, port)
if HAVE_SSL and not self.client:
# Is MongoDB configured for SSL?
self.client = _connect(host, port, **_SSL_OPTIONS)
if self.client:
self.ssl = True
self.ssl_client_options = _SSL_OPTIONS
self.ssl_certfile = True
if _SSL_OPTIONS.get('ssl_cert_reqs') == ssl.CERT_NONE:
self.ssl_cert_none = True
if self.client:
self.connected = True
ismaster = self.client.admin.command('ismaster')
if 'setName' in ismaster:
self.replica_set_name = ismaster['setName']
self.is_rs = True
# It doesn't matter which member we use as the seed here.
self.client = pymongo.MongoClient(
host,
port,
replicaSet=self.replica_set_name,
**self.ssl_client_options)
# Get the authoritative ismaster result from the primary.
self.ismaster = self.client.admin.command('ismaster')
nodes = [partition_node(node.lower())
for node in self.ismaster.get('hosts', [])]
nodes.extend([partition_node(node.lower())
for node in self.ismaster.get('passives', [])])
nodes.extend([partition_node(node.lower())
for node in self.ismaster.get('arbiters', [])])
self.nodes = set(nodes)
else:
self.ismaster = ismaster
self.nodes = set([(host, port)])
self.w = len(self.ismaster.get("hosts", [])) or 1
self.version = Version.from_client(self.client)
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
self.user_provided = self._check_user_provided()
if not self.user_provided:
roles = {}
if self.version.at_least(2, 5, 3, -1):
roles = {'roles': ['root']}
self.client.admin.add_user(db_user, db_pwd, **roles)
self.client.admin.authenticate(db_user, db_pwd)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
else:
params = self.cmd_line['parsed'].get('setParameter', {})
if params.get('enableTestCommands') == '1':
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.rs_client = None
self.cmd_line = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.is_rs = False
self.has_ipv6 = False
try:
client = pymongo.MongoClient(host, port,
serverSelectionTimeoutMS=100)
client.admin.command('ismaster') # Can we connect?
# If so, then reset client to defaults.
self.client = pymongo.MongoClient(host, port)
except pymongo.errors.ConnectionFailure:
self.client = None
else:
self.connected = True
self.ismaster = self.client.admin.command('ismaster')
self.w = len(self.ismaster.get("hosts", [])) or 1
self.nodes = set([(host, port)])
self.replica_set_name = self.ismaster.get('setName', '')
self.rs_client = None
self.version = Version.from_client(self.client)
if self.replica_set_name:
self.is_rs = True
self.rs_client = pymongo.MongoClient(
pair, replicaSet=self.replica_set_name)
nodes = [partition_node(node)
for node in self.ismaster.get('hosts', [])]
nodes.extend([partition_node(node)
for node in self.ismaster.get('passives', [])])
nodes.extend([partition_node(node)
for node in self.ismaster.get('arbiters', [])])
self.nodes = set(nodes)
self.rs_or_standalone_client = self.rs_client or self.client
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
self.user_provided = self._check_user_provided()
if not self.user_provided:
roles = {}
if self.version.at_least(2, 5, 3, -1):
roles = {'roles': ['root']}
self.client.admin.add_user(db_user, db_pwd, **roles)
self.client.admin.authenticate(db_user, db_pwd)
if self.rs_client:
self.rs_client.admin.authenticate(db_user, db_pwd)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
def _all_users(db):
if Version.from_client(db.client).at_least(2, 5, 3, -1):
return set(u['user'] for u in db.command('usersInfo').get('users', []))
else:
return set(u['user'] for u in db.system.users.find())
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.cmd_line = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.is_rs = False
self.has_ipv6 = False
self.ssl = False
self.ssl_cert_none = False
self.ssl_certfile = False
self.server_is_resolvable = is_server_resolvable()
self.ssl_client_options = {}
self.sessions_enabled = False
self.client = _connect(host, port)
if HAVE_SSL and not self.client:
# Is MongoDB configured for SSL?
self.client = _connect(host, port, **_SSL_OPTIONS)
if self.client:
self.ssl = True
self.ssl_client_options = _SSL_OPTIONS
self.ssl_certfile = True
if _SSL_OPTIONS.get('ssl_cert_reqs') == ssl.CERT_NONE:
self.ssl_cert_none = True
if self.client:
self.connected = True
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
if not self._check_user_provided():
self.client.admin.add_user(db_user, db_pwd, roles=['root'])
self.client = _connect(host,
port,
username=db_user,
password=db_pwd,
replicaSet=self.replica_set_name,
**self.ssl_client_options)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
self.ismaster = ismaster = self.client.admin.command('isMaster')
self.sessions_enabled = 'logicalSessionTimeoutMinutes' in ismaster
if 'setName' in ismaster:
self.replica_set_name = ismaster['setName']
self.is_rs = True
if self.auth_enabled:
# It doesn't matter which member we use as the seed here.
self.client = pymongo.MongoClient(
host,
port,
username=db_user,
password=db_pwd,
replicaSet=self.replica_set_name,
**self.ssl_client_options)
else:
self.client = pymongo.MongoClient(
host,
port,
replicaSet=self.replica_set_name,
**self.ssl_client_options)
# Get the authoritative ismaster result from the primary.
self.ismaster = self.client.admin.command('ismaster')
nodes = [
partition_node(node.lower())
for node in self.ismaster.get('hosts', [])
]
nodes.extend([
partition_node(node.lower())
for node in self.ismaster.get('passives', [])
])
nodes.extend([
partition_node(node.lower())
for node in self.ismaster.get('arbiters', [])
])
self.nodes = set(nodes)
else:
self.ismaster = ismaster
self.nodes = set([(host, port)])
self.w = len(self.ismaster.get("hosts", [])) or 1
self.version = Version.from_client(self.client)
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
else:
params = self.cmd_line['parsed'].get('setParameter', {})
if params.get('enableTestCommands') == '1':
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
def test_mongodb_x509_auth(self):
# Expects the server to be running with the server.pem, ca.pem
# and crl.pem provided in mongodb and the server tests as well as
# --auth
#
# --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem
# --sslCAFile=/path/to/pymongo/test/certificates/ca.pem
# --sslCRLFile=/path/to/pymongo/test/certificates/crl.pem
# --auth
if not CERT_SSL:
raise SkipTest("No mongod available over SSL with certs")
if not Version.from_client(ssl_client).at_least(2, 5, 3, -1):
raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer")
if not server_started_with_auth(ssl_client):
raise SkipTest('Authentication is not enabled on server')
self.addCleanup(ssl_client['$external'].logout)
self.addCleanup(remove_all_users, ssl_client['$external'])
# Give admin all necessary privileges.
ssl_client['$external'].add_user(MONGODB_X509_USERNAME, roles=[
{'role': 'readWriteAnyDatabase', 'db': 'admin'},
{'role': 'userAdminAnyDatabase', 'db': 'admin'}])
coll = ssl_client.pymongo_test.test
self.assertRaises(OperationFailure, coll.count)
self.assertTrue(ssl_client.admin.authenticate(
MONGODB_X509_USERNAME, mechanism='MONGODB-X509'))
coll.drop()
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' % (
quote_plus(MONGODB_X509_USERNAME), host, port))
# SSL options aren't supported in the URI...
self.assertTrue(MongoClient(uri,
ssl=True, ssl_certfile=CLIENT_PEM))
# Should require a username
uri = ('mongodb://%s:%d/?authMechanism=MONGODB-X509' % (host,
port))
client_bad = MongoClient(uri, ssl=True, ssl_certfile=CLIENT_PEM)
self.assertRaises(OperationFailure,
client_bad.pymongo_test.test.delete_one, {})
# Auth should fail if username and certificate do not match
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' % (
quote_plus("not the username"), host, port))
bad_client = MongoClient(uri, ssl=True, ssl_certfile=CLIENT_PEM)
with self.assertRaises(OperationFailure):
bad_client.pymongo_test.test.find_one()
self.assertRaises(OperationFailure, ssl_client.admin.authenticate,
"not the username",
mechanism="MONGODB-X509")
# Invalid certificate (using CA certificate as client certificate)
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' % (
quote_plus(MONGODB_X509_USERNAME), host, port))
# These tests will raise SSLError (>= 3.2) or ConnectionFailure
# (2.x) depending on where OpenSSL first sees the PEM file.
try:
connected(MongoClient(uri, ssl=True, ssl_certfile=CA_PEM,
serverSelectionTimeoutMS=100))
except (ssl.SSLError, ConnectionFailure):
pass
else:
self.fail("Invalid certificate accepted.")
try:
connected(MongoClient(pair, ssl=True, ssl_certfile=CA_PEM,
serverSelectionTimeoutMS=100))
except (ssl.SSLError, ConnectionFailure):
pass
else:
self.fail("Invalid certificate accepted.")
class ClientContext(object):
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connection_attempts = []
self.connected = False
self.w = None
self.nodes = set()
self.replica_set_name = None
self.cmd_line = None
self.server_status = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.server_parameters = None
self.is_mongos = False
self.mongoses = []
self.is_rs = False
self.has_ipv6 = False
self.tls = False
self.ssl_certfile = False
self.server_is_resolvable = is_server_resolvable()
self.default_client_options = {}
self.sessions_enabled = False
self.client = None
self.conn_lock = threading.Lock()
self.is_data_lake = False
if COMPRESSORS:
self.default_client_options["compressors"] = COMPRESSORS
if MONGODB_API_VERSION:
server_api = ServerApi(MONGODB_API_VERSION)
self.default_client_options["server_api"] = server_api
@property
def client_options(self):
"""Return the MongoClient options for creating a duplicate client."""
opts = client_context.default_client_options.copy()
if client_context.auth_enabled:
opts['username'] = db_user
opts['password'] = db_pwd
if self.replica_set_name:
opts['replicaSet'] = self.replica_set_name
return opts
@property
def ismaster(self):
return self.client.admin.command('isMaster')
def _connect(self, host, port, **kwargs):
# Jython takes a long time to connect.
if sys.platform.startswith('java'):
timeout_ms = 10000
else:
timeout_ms = 5000
kwargs.update(self.default_client_options)
client = pymongo.MongoClient(host,
port,
serverSelectionTimeoutMS=timeout_ms,
**kwargs)
try:
try:
client.admin.command('isMaster') # Can we connect?
except pymongo.errors.OperationFailure as exc:
# SERVER-32063
self.connection_attempts.append(
'connected client %r, but isMaster failed: %s' %
(client, exc))
else:
self.connection_attempts.append(
'successfully connected client %r' % (client, ))
# If connected, then return client with default timeout
return pymongo.MongoClient(host, port, **kwargs)
except pymongo.errors.ConnectionFailure as exc:
self.connection_attempts.append('failed to connect client %r: %s' %
(client, exc))
return None
finally:
client.close()
def _init_client(self):
self.client = self._connect(host, port)
if self.client is not None:
# Return early when connected to dataLake as mongohoused does not
# support the getCmdLineOpts command and is tested without TLS.
build_info = self.client.admin.command('buildInfo')
if 'dataLake' in build_info:
self.is_data_lake = True
self.auth_enabled = True
self.client = self._connect(host,
port,
username=db_user,
password=db_pwd)
self.connected = True
return
if HAVE_SSL and not self.client:
# Is MongoDB configured for SSL?
self.client = self._connect(host, port, **TLS_OPTIONS)
if self.client:
self.tls = True
self.default_client_options.update(TLS_OPTIONS)
self.ssl_certfile = True
if self.client:
self.connected = True
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
if not self._check_user_provided():
_create_user(self.client.admin, db_user, db_pwd)
self.client = self._connect(host,
port,
username=db_user,
password=db_pwd,
replicaSet=self.replica_set_name,
**self.default_client_options)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
self.server_status = self.client.admin.command('serverStatus')
if self.storage_engine == "mmapv1":
# MMAPv1 does not support retryWrites=True.
self.default_client_options['retryWrites'] = False
ismaster = self.ismaster
self.sessions_enabled = 'logicalSessionTimeoutMinutes' in ismaster
if 'setName' in ismaster:
self.replica_set_name = str(ismaster['setName'])
self.is_rs = True
if self.auth_enabled:
# It doesn't matter which member we use as the seed here.
self.client = pymongo.MongoClient(
host,
port,
username=db_user,
password=db_pwd,
replicaSet=self.replica_set_name,
**self.default_client_options)
else:
self.client = pymongo.MongoClient(
host,
port,
replicaSet=self.replica_set_name,
**self.default_client_options)
# Get the authoritative ismaster result from the primary.
ismaster = self.ismaster
nodes = [
partition_node(node.lower())
for node in ismaster.get('hosts', [])
]
nodes.extend([
partition_node(node.lower())
for node in ismaster.get('passives', [])
])
nodes.extend([
partition_node(node.lower())
for node in ismaster.get('arbiters', [])
])
self.nodes = set(nodes)
else:
self.nodes = set([(host, port)])
self.w = len(ismaster.get("hosts", [])) or 1
self.version = Version.from_client(self.client)
self.server_parameters = self.client.admin.command(
'getParameter', '*')
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
else:
params = self.cmd_line['parsed'].get('setParameter', {})
if params.get('enableTestCommands') == '1':
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
if self.is_mongos:
# Check for another mongos on the next port.
address = self.client.address
next_address = address[0], address[1] + 1
self.mongoses.append(address)
mongos_client = self._connect(*next_address,
**self.default_client_options)
if mongos_client:
ismaster = mongos_client.admin.command('ismaster')
if ismaster.get('msg') == 'isdbgrid':
self.mongoses.append(next_address)
def init(self):
with self.conn_lock:
if not self.client and not self.connection_attempts:
self._init_client()
def connection_attempt_info(self):
return '\n'.join(self.connection_attempts)
@property
def host(self):
if self.is_rs:
primary = self.client.primary
return str(primary[0]) if primary is not None else host
return host
@property
def port(self):
if self.is_rs:
primary = self.client.primary
return primary[1] if primary is not None else port
return port
@property
def pair(self):
return "%s:%d" % (self.host, self.port)
@property
def has_secondaries(self):
if not self.client:
return False
return bool(len(self.client.secondaries))
@property
def storage_engine(self):
try:
return self.server_status.get("storageEngine", {}).get("name")
except AttributeError:
# Raised if self.server_status is None.
return None
def _check_user_provided(self):
"""Return True if db_user/db_password is already an admin user."""
client = pymongo.MongoClient(host,
port,
username=db_user,
password=db_pwd,
serverSelectionTimeoutMS=100,
**self.default_client_options)
try:
return db_user in _all_users(client.admin)
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 18 or 'auth fails' in msg:
# Auth failed.
return False
else:
raise
def _server_started_with_auth(self):
# MongoDB >= 2.0
if 'parsed' in self.cmd_line:
parsed = self.cmd_line['parsed']
# MongoDB >= 2.6
if 'security' in parsed:
security = parsed['security']
# >= rc3
if 'authorization' in security:
return security['authorization'] == 'enabled'
# < rc3
return (security.get('auth', False)
or bool(security.get('keyFile')))
return parsed.get('auth', False) or bool(parsed.get('keyFile'))
# Legacy
argv = self.cmd_line['argv']
return '--auth' in argv or '--keyFile' in argv
def _server_started_with_ipv6(self):
if not socket.has_ipv6:
return False
if 'parsed' in self.cmd_line:
if not self.cmd_line['parsed'].get('net', {}).get('ipv6'):
return False
else:
if '--ipv6' not in self.cmd_line['argv']:
return False
# The server was started with --ipv6. Is there an IPv6 route to it?
try:
for info in socket.getaddrinfo(self.host, self.port):
if info[0] == socket.AF_INET6:
return True
except socket.error:
pass
return False
def _require(self, condition, msg, func=None):
def make_wrapper(f):
@wraps(f)
def wrap(*args, **kwargs):
self.init()
# Always raise SkipTest if we can't connect to MongoDB
if not self.connected:
raise SkipTest("Cannot connect to MongoDB on %s" %
(self.pair, ))
if condition():
return f(*args, **kwargs)
raise SkipTest(msg)
return wrap
if func is None:
def decorate(f):
return make_wrapper(f)
return decorate
return make_wrapper(func)
def create_user(self, dbname, user, pwd=None, roles=None, **kwargs):
kwargs['writeConcern'] = {'w': self.w}
return _create_user(self.client[dbname], user, pwd, roles, **kwargs)
def drop_user(self, dbname, user):
self.client[dbname].command('dropUser',
user,
writeConcern={'w': self.w})
def require_connection(self, func):
"""Run a test only if we can connect to MongoDB."""
return self._require(
lambda: True, # _require checks if we're connected
"Cannot connect to MongoDB on %s" % (self.pair, ),
func=func)
def require_no_mmap(self, func):
"""Run a test only if the server is not using the MMAPv1 storage
engine. Only works for standalone and replica sets; tests are
run regardless of storage engine on sharded clusters. """
def is_not_mmap():
if self.is_mongos:
return True
return self.storage_engine != 'mmapv1'
return self._require(is_not_mmap,
"Storage engine must not be MMAPv1",
func=func)
def require_version_min(self, *ver):
"""Run a test only if the server version is at least ``version``."""
other_version = Version(*ver)
return self._require(
lambda: self.version >= other_version,
"Server version must be at least %s" % str(other_version))
def require_version_max(self, *ver):
"""Run a test only if the server version is at most ``version``."""
other_version = Version(*ver)
return self._require(
lambda: self.version <= other_version,
"Server version must be at most %s" % str(other_version))
def require_auth(self, func):
"""Run a test only if the server is running with auth enabled."""
return self.check_auth_with_sharding(
self._require(lambda: self.auth_enabled,
"Authentication is not enabled on the server",
func=func))
def require_no_auth(self, func):
"""Run a test only if the server is running without auth enabled."""
return self._require(
lambda: not self.auth_enabled,
"Authentication must not be enabled on the server",
func=func)
def require_replica_set(self, func):
"""Run a test only if the client is connected to a replica set."""
return self._require(lambda: self.is_rs,
"Not connected to a replica set",
func=func)
def require_secondaries_count(self, count):
"""Run a test only if the client is connected to a replica set that has
`count` secondaries.
"""
def sec_count():
return 0 if not self.client else len(self.client.secondaries)
return self._require(lambda: sec_count() >= count,
"Not enough secondaries available")
def require_no_replica_set(self, func):
"""Run a test if the client is *not* connected to a replica set."""
return self._require(
lambda: not self.is_rs,
"Connected to a replica set, not a standalone mongod",
func=func)
def require_ipv6(self, func):
"""Run a test only if the client can connect to a server via IPv6."""
return self._require(lambda: self.has_ipv6, "No IPv6", func=func)
def require_no_mongos(self, func):
"""Run a test only if the client is not connected to a mongos."""
return self._require(lambda: not self.is_mongos,
"Must be connected to a mongod, not a mongos",
func=func)
def require_mongos(self, func):
"""Run a test only if the client is connected to a mongos."""
return self._require(lambda: self.is_mongos,
"Must be connected to a mongos",
func=func)
def require_multiple_mongoses(self, func):
"""Run a test only if the client is connected to a sharded cluster
that has 2 mongos nodes."""
return self._require(lambda: len(self.mongoses) > 1,
"Must have multiple mongoses available",
func=func)
def require_standalone(self, func):
"""Run a test only if the client is connected to a standalone."""
return self._require(lambda: not (self.is_mongos or self.is_rs),
"Must be connected to a standalone",
func=func)
def require_no_standalone(self, func):
"""Run a test only if the client is not connected to a standalone."""
return self._require(lambda: self.is_mongos or self.is_rs,
"Must be connected to a replica set or mongos",
func=func)
def check_auth_with_sharding(self, func):
"""Skip a test when connected to mongos < 2.0 and running with auth."""
condition = lambda: not (self.auth_enabled and self.is_mongos and self.
version < (2, ))
return self._require(condition,
"Auth with sharding requires MongoDB >= 2.0.0",
func=func)
def is_topology_type(self, topologies):
if 'single' in topologies and not (self.is_mongos or self.is_rs):
return True
if 'replicaset' in topologies and self.is_rs:
return True
if 'sharded' in topologies and self.is_mongos:
return True
if 'sharded-replicaset' in topologies and self.is_mongos:
shards = list(client_context.client.config.shards.find())
for shard in shards:
# For a 3-member RS-backed sharded cluster, shard['host']
# will be 'replicaName/ip1:port1,ip2:port2,ip3:port3'
# Otherwise it will be 'ip1:port1'
host_spec = shard['host']
if not len(host_spec.split('/')) > 1:
return False
return True
return False
def require_cluster_type(self, topologies=[]):
"""Run a test only if the client is connected to a cluster that
conforms to one of the specified topologies. Acceptable topologies
are 'single', 'replicaset', and 'sharded'."""
def _is_valid_topology():
return self.is_topology_type(topologies)
return self._require(_is_valid_topology,
"Cluster type not in %s" % (topologies))
def require_test_commands(self, func):
"""Run a test only if the server has test commands enabled."""
return self._require(lambda: self.test_commands_enabled,
"Test commands must be enabled",
func=func)
def require_failCommand_fail_point(self, func):
"""Run a test only if the server supports the failCommand fail
point."""
return self._require(lambda: self.supports_failCommand_fail_point,
"failCommand fail point must be supported",
func=func)
def require_failCommand_appName(self, func):
"""Run a test only if the server supports the failCommand appName."""
# SERVER-47195
return self._require(lambda:
(self.test_commands_enabled and self.version >=
(4, 4, -1)),
"failCommand appName must be supported",
func=func)
def require_tls(self, func):
"""Run a test only if the client can connect over TLS."""
return self._require(lambda: self.tls,
"Must be able to connect via TLS",
func=func)
def require_no_tls(self, func):
"""Run a test only if the client can connect over TLS."""
return self._require(lambda: not self.tls,
"Must be able to connect without TLS",
func=func)
def require_ssl_certfile(self, func):
"""Run a test only if the client can connect with ssl_certfile."""
return self._require(lambda: self.ssl_certfile,
"Must be able to connect with ssl_certfile",
func=func)
def require_server_resolvable(self, func):
"""Run a test only if the hostname 'server' is resolvable."""
return self._require(lambda: self.server_is_resolvable,
"No hosts entry for 'server'. Cannot validate "
"hostname in the certificate",
func=func)
def require_sessions(self, func):
"""Run a test only if the deployment supports sessions."""
return self._require(lambda: self.sessions_enabled,
"Sessions not supported",
func=func)
def supports_retryable_writes(self):
if self.storage_engine == 'mmapv1':
return False
if not self.sessions_enabled:
return False
if self.version.at_least(3, 6):
return self.is_mongos or self.is_rs
return False
def require_retryable_writes(self, func):
"""Run a test only if the deployment supports retryable writes."""
return self._require(self.supports_retryable_writes,
"This server does not support retryable writes",
func=func)
def supports_transactions(self):
if self.storage_engine == 'mmapv1':
return False
if self.version.at_least(4, 1, 8):
return self.is_mongos or self.is_rs
if self.version.at_least(4, 0):
return self.is_rs
return False
def require_transactions(self, func):
"""Run a test only if the deployment might support transactions.
*Might* because this does not test the storage engine or FCV.
"""
return self._require(self.supports_transactions,
"Transactions are not supported",
func=func)
def require_no_api_version(self, func):
"""Skip this test when testing with requireApiVersion."""
return self._require(lambda: not MONGODB_API_VERSION,
"This test does not work with requireApiVersion",
func=func)
def mongos_seeds(self):
return ','.join('%s:%s' % address for address in self.mongoses)
@property
def supports_failCommand_fail_point(self):
"""Does the server support the failCommand fail point?"""
if self.is_mongos:
return (self.version.at_least(4, 1, 5)
and self.test_commands_enabled)
else:
return (self.version.at_least(4, 0) and self.test_commands_enabled)
@property
def requires_hint_with_min_max_queries(self):
"""Does the server require a hint with min/max queries."""
# Changed in SERVER-39567.
return self.version.at_least(4, 1, 10)
class ClientContext(object):
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.cmd_line = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.is_rs = False
self.has_ipv6 = False
self.ssl = False
self.ssl_cert_none = False
self.ssl_certfile = False
self.server_is_resolvable = is_server_resolvable()
self.ssl_client_options = {}
self.client = _connect(host, port)
if HAVE_SSL and not self.client:
# Is MongoDB configured for SSL?
self.client = _connect(host, port, **_SSL_OPTIONS)
if self.client:
self.ssl = True
self.ssl_client_options = _SSL_OPTIONS
self.ssl_certfile = True
if _SSL_OPTIONS.get('ssl_cert_reqs') == ssl.CERT_NONE:
self.ssl_cert_none = True
if self.client:
self.connected = True
ismaster = self.client.admin.command('ismaster')
if 'setName' in ismaster:
self.replica_set_name = ismaster['setName']
self.is_rs = True
# It doesn't matter which member we use as the seed here.
self.client = pymongo.MongoClient(
host,
port,
replicaSet=self.replica_set_name,
**self.ssl_client_options)
# Get the authoritative ismaster result from the primary.
self.ismaster = self.client.admin.command('ismaster')
nodes = [
partition_node(node.lower())
for node in self.ismaster.get('hosts', [])
]
nodes.extend([
partition_node(node.lower())
for node in self.ismaster.get('passives', [])
])
nodes.extend([
partition_node(node.lower())
for node in self.ismaster.get('arbiters', [])
])
self.nodes = set(nodes)
else:
self.ismaster = ismaster
self.nodes = set([(host, port)])
self.w = len(self.ismaster.get("hosts", [])) or 1
self.version = Version.from_client(self.client)
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
self.user_provided = self._check_user_provided()
if not self.user_provided:
roles = {}
if self.version.at_least(2, 5, 3, -1):
roles = {'roles': ['root']}
self.client.admin.add_user(db_user, db_pwd, **roles)
self.client = _connect(host,
port,
username=db_user,
password=db_pwd,
replicaSet=self.replica_set_name,
**self.ssl_client_options)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
else:
params = self.cmd_line['parsed'].get('setParameter', {})
if params.get('enableTestCommands') == '1':
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
@property
def host(self):
if self.is_rs:
primary = self.client.primary
return primary[0] if primary is not None else host
return host
@property
def port(self):
if self.is_rs:
primary = self.client.primary
return primary[1] if primary is not None else port
return port
@property
def pair(self):
return "%s:%d" % (self.host, self.port)
@property
def has_secondaries(self):
if not self.client:
return False
return bool(len(self.client.secondaries))
def _check_user_provided(self):
"""Return True if db_user/db_password is already an admin user."""
client = pymongo.MongoClient(host,
port,
username=db_user,
password=db_pwd,
serverSelectionTimeoutMS=100,
**self.ssl_client_options)
try:
return db_user in _all_users(client.admin)
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 18 or 'auth fails' in msg:
# Auth failed.
return False
else:
raise
def _server_started_with_auth(self):
# MongoDB >= 2.0
if 'parsed' in self.cmd_line:
parsed = self.cmd_line['parsed']
# MongoDB >= 2.6
if 'security' in parsed:
security = parsed['security']
# >= rc3
if 'authorization' in security:
return security['authorization'] == 'enabled'
# < rc3
return (security.get('auth', False)
or bool(security.get('keyFile')))
return parsed.get('auth', False) or bool(parsed.get('keyFile'))
# Legacy
argv = self.cmd_line['argv']
return '--auth' in argv or '--keyFile' in argv
def _server_started_with_ipv6(self):
if not socket.has_ipv6:
return False
if 'parsed' in self.cmd_line:
if not self.cmd_line['parsed'].get('net', {}).get('ipv6'):
return False
else:
if '--ipv6' not in self.cmd_line['argv']:
return False
# The server was started with --ipv6. Is there an IPv6 route to it?
try:
for info in socket.getaddrinfo(self.host, self.port):
if info[0] == socket.AF_INET6:
return True
except socket.error:
pass
return False
def _require(self, condition, msg, func=None):
def make_wrapper(f):
@wraps(f)
def wrap(*args, **kwargs):
# Always raise SkipTest if we can't connect to MongoDB
if not self.connected:
raise SkipTest("Cannot connect to MongoDB on %s" %
(self.pair, ))
if condition:
return f(*args, **kwargs)
raise SkipTest(msg)
return wrap
if func is None:
def decorate(f):
return make_wrapper(f)
return decorate
return make_wrapper(func)
def require_connection(self, func):
"""Run a test only if we can connect to MongoDB."""
return self._require(self.connected,
"Cannot connect to MongoDB on %s" % (self.pair, ),
func=func)
def require_version_min(self, *ver):
"""Run a test only if the server version is at least ``version``."""
other_version = Version(*ver)
return self._require(
self.version >= other_version,
"Server version must be at least %s" % str(other_version))
def require_version_max(self, *ver):
"""Run a test only if the server version is at most ``version``."""
other_version = Version(*ver)
return self._require(
self.version <= other_version,
"Server version must be at most %s" % str(other_version))
def require_auth(self, func):
"""Run a test only if the server is running with auth enabled."""
return self.check_auth_with_sharding(
self._require(self.auth_enabled,
"Authentication is not enabled on the server",
func=func))
def require_no_auth(self, func):
"""Run a test only if the server is running without auth enabled."""
return self._require(
not self.auth_enabled,
"Authentication must not be enabled on the server",
func=func)
def require_replica_set(self, func):
"""Run a test only if the client is connected to a replica set."""
return self._require(self.is_rs,
"Not connected to a replica set",
func=func)
def require_secondaries_count(self, count):
"""Run a test only if the client is connected to a replica set that has
`count` secondaries.
"""
sec_count = 0 if not self.client else len(self.client.secondaries)
return self._require(
sec_count >= count,
"Need %d secondaries, %d available" % (count, sec_count))
def require_no_replica_set(self, func):
"""Run a test if the client is *not* connected to a replica set."""
return self._require(
not self.is_rs,
"Connected to a replica set, not a standalone mongod",
func=func)
def require_ipv6(self, func):
"""Run a test only if the client can connect to a server via IPv6."""
return self._require(self.has_ipv6, "No IPv6", func=func)
def require_no_mongos(self, func):
"""Run a test only if the client is not connected to a mongos."""
return self._require(not self.is_mongos,
"Must be connected to a mongod, not a mongos",
func=func)
def require_mongos(self, func):
"""Run a test only if the client is connected to a mongos."""
return self._require(self.is_mongos,
"Must be connected to a mongos",
func=func)
def check_auth_with_sharding(self, func):
"""Skip a test when connected to mongos < 2.0 and running with auth."""
condition = not (self.auth_enabled and self.is_mongos
and self.version < (2, ))
return self._require(condition,
"Auth with sharding requires MongoDB >= 2.0.0",
func=func)
def require_test_commands(self, func):
"""Run a test only if the server has test commands enabled."""
return self._require(self.test_commands_enabled,
"Test commands must be enabled",
func=func)
def require_ssl(self, func):
"""Run a test only if the client can connect over SSL."""
return self._require(self.ssl,
"Must be able to connect via SSL",
func=func)
def require_no_ssl(self, func):
"""Run a test only if the client can connect over SSL."""
return self._require(not self.ssl,
"Must be able to connect without SSL",
func=func)
def require_ssl_cert_none(self, func):
"""Run a test only if the client can connect with ssl.CERT_NONE."""
return self._require(self.ssl_cert_none,
"Must be able to connect with ssl.CERT_NONE",
func=func)
def require_ssl_certfile(self, func):
"""Run a test only if the client can connect with ssl_certfile."""
return self._require(self.ssl_certfile,
"Must be able to connect with ssl_certfile",
func=func)
def require_server_resolvable(self, func):
"""Run a test only if the hostname 'server' is resolvable."""
return self._require(self.server_is_resolvable,
"No hosts entry for 'server'. Cannot validate "
"hostname in the certificate",
func=func)
class ClientContext(object):
def __init__(self):
"""Create a client and grab essential information from the server."""
self.connection_attempts = []
self.connected = False
self.ismaster = {}
self.w = None
self.nodes = set()
self.replica_set_name = None
self.cmd_line = None
self.server_status = None
self.version = Version(-1) # Needs to be comparable with Version
self.auth_enabled = False
self.test_commands_enabled = False
self.is_mongos = False
self.mongoses = []
self.is_rs = False
self.has_ipv6 = False
self.ssl = False
self.ssl_cert_none = False
self.ssl_certfile = False
self.server_is_resolvable = is_server_resolvable()
self.default_client_options = {}
self.sessions_enabled = False
self.client = None
self.conn_lock = threading.Lock()
if COMPRESSORS:
self.default_client_options["compressors"] = COMPRESSORS
def _connect(self, host, port, **kwargs):
# Jython takes a long time to connect.
if sys.platform.startswith('java'):
timeout_ms = 10000
else:
timeout_ms = 5000
if COMPRESSORS:
kwargs["compressors"] = COMPRESSORS
client = pymongo.MongoClient(
host, port, serverSelectionTimeoutMS=timeout_ms, **kwargs)
try:
try:
client.admin.command('isMaster') # Can we connect?
except pymongo.errors.OperationFailure as exc:
# SERVER-32063
self.connection_attempts.append(
'connected client %r, but isMaster failed: %s' % (
client, exc))
else:
self.connection_attempts.append(
'successfully connected client %r' % (client,))
# If connected, then return client with default timeout
return pymongo.MongoClient(host, port, **kwargs)
except pymongo.errors.ConnectionFailure as exc:
self.connection_attempts.append(
'failed to connect client %r: %s' % (client, exc))
return None
def _init_client(self):
self.client = self._connect(host, port)
if HAVE_SSL and not self.client:
# Is MongoDB configured for SSL?
self.client = self._connect(host, port, **_SSL_OPTIONS)
if self.client:
self.ssl = True
self.default_client_options.update(_SSL_OPTIONS)
self.ssl_certfile = True
if _SSL_OPTIONS.get('ssl_cert_reqs') == ssl.CERT_NONE:
self.ssl_cert_none = True
if self.client:
self.connected = True
try:
self.cmd_line = self.client.admin.command('getCmdLineOpts')
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 13 or 'unauthorized' in msg or 'login' in msg:
# Unauthorized.
self.auth_enabled = True
else:
raise
else:
self.auth_enabled = self._server_started_with_auth()
if self.auth_enabled:
# See if db_user already exists.
if not self._check_user_provided():
_create_user(self.client.admin, db_user, db_pwd)
self.client = self._connect(
host, port, username=db_user, password=db_pwd,
replicaSet=self.replica_set_name,
**self.default_client_options)
# May not have this if OperationFailure was raised earlier.
self.cmd_line = self.client.admin.command('getCmdLineOpts')
self.server_status = self.client.admin.command('serverStatus')
self.ismaster = ismaster = self.client.admin.command('isMaster')
self.sessions_enabled = 'logicalSessionTimeoutMinutes' in ismaster
if 'setName' in ismaster:
self.replica_set_name = str(ismaster['setName'])
self.is_rs = True
if self.auth_enabled:
# It doesn't matter which member we use as the seed here.
self.client = pymongo.MongoClient(
host,
port,
username=db_user,
password=db_pwd,
replicaSet=self.replica_set_name,
**self.default_client_options)
else:
self.client = pymongo.MongoClient(
host,
port,
replicaSet=self.replica_set_name,
**self.default_client_options)
# Get the authoritative ismaster result from the primary.
self.ismaster = self.client.admin.command('ismaster')
nodes = [partition_node(node.lower())
for node in self.ismaster.get('hosts', [])]
nodes.extend([partition_node(node.lower())
for node in self.ismaster.get('passives', [])])
nodes.extend([partition_node(node.lower())
for node in self.ismaster.get('arbiters', [])])
self.nodes = set(nodes)
else:
self.ismaster = ismaster
self.nodes = set([(host, port)])
self.w = len(self.ismaster.get("hosts", [])) or 1
self.version = Version.from_client(self.client)
if 'enableTestCommands=1' in self.cmd_line['argv']:
self.test_commands_enabled = True
elif 'parsed' in self.cmd_line:
params = self.cmd_line['parsed'].get('setParameter', [])
if 'enableTestCommands=1' in params:
self.test_commands_enabled = True
else:
params = self.cmd_line['parsed'].get('setParameter', {})
if params.get('enableTestCommands') == '1':
self.test_commands_enabled = True
self.is_mongos = (self.ismaster.get('msg') == 'isdbgrid')
self.has_ipv6 = self._server_started_with_ipv6()
if self.is_mongos:
# Check for another mongos on the next port.
address = self.client.address
next_address = address[0], address[1] + 1
self.mongoses.append(address)
mongos_client = self._connect(*next_address,
**self.default_client_options)
if mongos_client:
ismaster = mongos_client.admin.command('ismaster')
if ismaster.get('msg') == 'isdbgrid':
self.mongoses.append(next_address)
def init(self):
with self.conn_lock:
if not self.client and not self.connection_attempts:
self._init_client()
def connection_attempt_info(self):
return '\n'.join(self.connection_attempts)
@property
def host(self):
if self.is_rs:
primary = self.client.primary
return str(primary[0]) if primary is not None else host
return host
@property
def port(self):
if self.is_rs:
primary = self.client.primary
return primary[1] if primary is not None else port
return port
@property
def pair(self):
return "%s:%d" % (self.host, self.port)
@property
def has_secondaries(self):
if not self.client:
return False
return bool(len(self.client.secondaries))
def _check_user_provided(self):
"""Return True if db_user/db_password is already an admin user."""
client = pymongo.MongoClient(
host, port,
username=db_user,
password=db_pwd,
serverSelectionTimeoutMS=100,
**self.default_client_options)
try:
return db_user in _all_users(client.admin)
except pymongo.errors.OperationFailure as e:
msg = e.details.get('errmsg', '')
if e.code == 18 or 'auth fails' in msg:
# Auth failed.
return False
else:
raise
def _server_started_with_auth(self):
# MongoDB >= 2.0
if 'parsed' in self.cmd_line:
parsed = self.cmd_line['parsed']
# MongoDB >= 2.6
if 'security' in parsed:
security = parsed['security']
# >= rc3
if 'authorization' in security:
return security['authorization'] == 'enabled'
# < rc3
return (security.get('auth', False) or
bool(security.get('keyFile')))
return parsed.get('auth', False) or bool(parsed.get('keyFile'))
# Legacy
argv = self.cmd_line['argv']
return '--auth' in argv or '--keyFile' in argv
def _server_started_with_ipv6(self):
if not socket.has_ipv6:
return False
if 'parsed' in self.cmd_line:
if not self.cmd_line['parsed'].get('net', {}).get('ipv6'):
return False
else:
if '--ipv6' not in self.cmd_line['argv']:
return False
# The server was started with --ipv6. Is there an IPv6 route to it?
try:
for info in socket.getaddrinfo(self.host, self.port):
if info[0] == socket.AF_INET6:
return True
except socket.error:
pass
return False
def _require(self, condition, msg, func=None):
def make_wrapper(f):
@wraps(f)
def wrap(*args, **kwargs):
self.init()
# Always raise SkipTest if we can't connect to MongoDB
if not self.connected:
raise SkipTest(
"Cannot connect to MongoDB on %s" % (self.pair,))
if condition():
return f(*args, **kwargs)
raise SkipTest(msg)
return wrap
if func is None:
def decorate(f):
return make_wrapper(f)
return decorate
return make_wrapper(func)
def create_user(self, dbname, user, pwd=None, roles=None, **kwargs):
kwargs['writeConcern'] = {'w': self.w}
return _create_user(self.client[dbname], user, pwd, roles, **kwargs)
def drop_user(self, dbname, user):
self.client[dbname].command(
'dropUser', user, writeConcern={'w': self.w})
def require_connection(self, func):
"""Run a test only if we can connect to MongoDB."""
return self._require(
lambda: True, # _require checks if we're connected
"Cannot connect to MongoDB on %s" % (self.pair,),
func=func)
def require_no_mmap(self, func):
"""Run a test only if the server is not using the MMAPv1 storage
engine. Only works for standalone and replica sets; tests are
run regardless of storage engine on sharded clusters. """
def is_not_mmap():
if self.is_mongos:
return True
try:
storage_engine = self.server_status.get(
'storageEngine').get('name')
except AttributeError:
# Raised if the storageEngine key does not exist or if
# self.server_status is None.
return False
return storage_engine != 'mmapv1'
return self._require(
is_not_mmap, "Storage engine must not be MMAPv1", func=func)
def require_version_min(self, *ver):
"""Run a test only if the server version is at least ``version``."""
other_version = Version(*ver)
return self._require(lambda: self.version >= other_version,
"Server version must be at least %s"
% str(other_version))
def require_version_max(self, *ver):
"""Run a test only if the server version is at most ``version``."""
other_version = Version(*ver)
return self._require(lambda: self.version <= other_version,
"Server version must be at most %s"
% str(other_version))
def require_auth(self, func):
"""Run a test only if the server is running with auth enabled."""
return self.check_auth_with_sharding(
self._require(lambda: self.auth_enabled,
"Authentication is not enabled on the server",
func=func))
def require_no_auth(self, func):
"""Run a test only if the server is running without auth enabled."""
return self._require(lambda: not self.auth_enabled,
"Authentication must not be enabled on the server",
func=func)
def require_replica_set(self, func):
"""Run a test only if the client is connected to a replica set."""
return self._require(lambda: self.is_rs,
"Not connected to a replica set",
func=func)
def require_secondaries_count(self, count):
"""Run a test only if the client is connected to a replica set that has
`count` secondaries.
"""
def sec_count():
return 0 if not self.client else len(self.client.secondaries)
return self._require(lambda: sec_count() >= count,
"Not enough secondaries available")
def require_no_replica_set(self, func):
"""Run a test if the client is *not* connected to a replica set."""
return self._require(
lambda: not self.is_rs,
"Connected to a replica set, not a standalone mongod",
func=func)
def require_ipv6(self, func):
"""Run a test only if the client can connect to a server via IPv6."""
return self._require(lambda: self.has_ipv6,
"No IPv6",
func=func)
def require_no_mongos(self, func):
"""Run a test only if the client is not connected to a mongos."""
return self._require(lambda: not self.is_mongos,
"Must be connected to a mongod, not a mongos",
func=func)
def require_mongos(self, func):
"""Run a test only if the client is connected to a mongos."""
return self._require(lambda: self.is_mongos,
"Must be connected to a mongos",
func=func)
def require_multiple_mongoses(self, func):
"""Run a test only if the client is connected to a sharded cluster
that has 2 mongos nodes."""
return self._require(lambda: len(self.mongoses) > 1,
"Must have multiple mongoses available",
func=func)
def require_standalone(self, func):
"""Run a test only if the client is connected to a standalone."""
return self._require(lambda: not (self.is_mongos or self.is_rs),
"Must be connected to a standalone",
func=func)
def require_no_standalone(self, func):
"""Run a test only if the client is not connected to a standalone."""
return self._require(lambda: self.is_mongos or self.is_rs,
"Must be connected to a replica set or mongos",
func=func)
def check_auth_with_sharding(self, func):
"""Skip a test when connected to mongos < 2.0 and running with auth."""
condition = lambda: not (self.auth_enabled and
self.is_mongos and self.version < (2,))
return self._require(condition,
"Auth with sharding requires MongoDB >= 2.0.0",
func=func)
def is_topology_type(self, topologies):
if 'single' in topologies and not (self.is_mongos or self.is_rs):
return True
if 'replicaset' in topologies and self.is_rs:
return True
if 'sharded' in topologies and self.is_mongos:
return True
return False
def require_cluster_type(self, topologies=[]):
"""Run a test only if the client is connected to a cluster that
conforms to one of the specified topologies. Acceptable topologies
are 'single', 'replicaset', and 'sharded'."""
def _is_valid_topology():
return self.is_topology_type(topologies)
return self._require(
_is_valid_topology,
"Cluster type not in %s" % (topologies))
def require_test_commands(self, func):
"""Run a test only if the server has test commands enabled."""
return self._require(lambda: self.test_commands_enabled,
"Test commands must be enabled",
func=func)
def require_ssl(self, func):
"""Run a test only if the client can connect over SSL."""
return self._require(lambda: self.ssl,
"Must be able to connect via SSL",
func=func)
def require_no_ssl(self, func):
"""Run a test only if the client can connect over SSL."""
return self._require(lambda: not self.ssl,
"Must be able to connect without SSL",
func=func)
def require_ssl_cert_none(self, func):
"""Run a test only if the client can connect with ssl.CERT_NONE."""
return self._require(lambda: self.ssl_cert_none,
"Must be able to connect with ssl.CERT_NONE",
func=func)
def require_ssl_certfile(self, func):
"""Run a test only if the client can connect with ssl_certfile."""
return self._require(lambda: self.ssl_certfile,
"Must be able to connect with ssl_certfile",
func=func)
def require_server_resolvable(self, func):
"""Run a test only if the hostname 'server' is resolvable."""
return self._require(lambda: self.server_is_resolvable,
"No hosts entry for 'server'. Cannot validate "
"hostname in the certificate",
func=func)
def require_sessions(self, func):
"""Run a test only if the deployment supports sessions."""
return self._require(lambda: self.sessions_enabled,
"Sessions not supported",
func=func)
def supports_transactions(self):
if self.version.at_least(4, 1, 8):
return self.is_mongos or self.is_rs
if self.version.at_least(4, 0):
return self.is_rs
return False
def require_transactions(self, func):
"""Run a test only if the deployment might support transactions.
*Might* because this does not test the storage engine or FCV.
"""
return self._require(self.supports_transactions,
"Transactions are not supported",
func=func)
def mongos_seeds(self):
return ','.join('%s:%s' % address for address in self.mongoses)
@property
def supports_reindex(self):
"""Does the connected server support reindex?"""
return not (self.version.at_least(4, 1, 0) and self.is_mongos)
@property
def supports_getpreverror(self):
"""Does the connected server support getpreverror?"""
return not (self.version.at_least(4, 1, 0) or self.is_mongos)
@property
def requires_hint_with_min_max_queries(self):
"""Does the server require a hint with min/max queries."""
# Changed in SERVER-39567.
return self.version.at_least(4, 1, 10)
def setup_version(self):
"""Set self.version to the server's version."""
self.version = Version.from_client(self.sync_cx)
