def make_server_description(server, hosts):
"""Make a ServerDescription from server info in a JSON test."""
server_type = server['type']
if server_type == "Unknown":
return ServerDescription(clean_node(server['address']), IsMaster({}))
ismaster_response = {'ok': True, 'hosts': hosts}
if server_type != "Standalone" and server_type != "Mongos":
ismaster_response['setName'] = "rs"
if server_type == "RSPrimary":
ismaster_response['ismaster'] = True
elif server_type == "RSSecondary":
ismaster_response['secondary'] = True
elif server_type == "Mongos":
ismaster_response['msg'] = 'isdbgrid'
ismaster_response['lastWrite'] = {
'lastWriteDate': make_last_write_date(server)
}
for field in 'maxWireVersion', 'tags', 'idleWritePeriodMillis':
if field in server:
ismaster_response[field] = server[field]
# Sets _last_update_time to now.
sd = ServerDescription(clean_node(server['address']),
IsMaster(ismaster_response),
round_trip_time=server['avg_rtt_ms'] / 1000.0)
if 'lastUpdateTime' in server:
sd._last_update_time = server['lastUpdateTime'] / 1000.0 # ms to sec.
return sd
def make_server_description(server, hosts):
"""Make ServerDescription from server info from JSON file."""
server_type = server['type']
if server_type == "Unknown":
return ServerDescription(clean_node(server['address']), IsMaster({}))
ismaster_response = {'ok': True, 'hosts': hosts}
if server_type != "Standalone" and server_type != "Mongos":
ismaster_response['setName'] = "rs"
if server_type == "RSPrimary":
ismaster_response['ismaster'] = True
elif server_type == "RSSecondary":
ismaster_response['secondary'] = True
elif server_type == "Mongos":
ismaster_response['msg'] = 'isdbgrid'
ismaster_response['lastWrite'] = {
'lastWriteDate': make_last_write_date(server)
}
for field in 'maxWireVersion', 'tags', 'idleWritePeriodMillis':
if field in server:
ismaster_response[field] = server[field]
# Sets _last_update_time to now.
sd = ServerDescription(clean_node(server['address']),
IsMaster(ismaster_response),
round_trip_time=server['avg_rtt_ms'])
sd._last_update_time = server['lastUpdateTime'] / 1000.0 # ms to sec.
return sd
def _handle_error(self, address, err_ctx):
if self._is_stale_error(address, err_ctx):
return
server = self._servers[address]
error = err_ctx.error
exc_type = type(error)
service_id = err_ctx.service_id
if (issubclass(exc_type, NetworkTimeout)
and err_ctx.completed_handshake):
# The socket has been closed. Don't reset the server.
# Server Discovery And Monitoring Spec: "When an application
# operation fails because of any network error besides a socket
# timeout...."
return
elif issubclass(exc_type, WriteError):
# Ignore writeErrors.
return
elif issubclass(exc_type, (NotPrimaryError, OperationFailure)):
# As per the SDAM spec if:
# - the server sees a "not primary" error, and
# - the server is not shutting down, and
# - the server version is >= 4.2, then
# we keep the existing connection pool, but mark the server type
# as Unknown and request an immediate check of the server.
# Otherwise, we clear the connection pool, mark the server as
# Unknown and request an immediate check of the server.
if hasattr(error, 'code'):
err_code = error.code
else:
err_code = error.details.get('code', -1)
if err_code in helpers._NOT_MASTER_CODES:
is_shutting_down = err_code in helpers._SHUTDOWN_CODES
# Mark server Unknown, clear the pool, and request check.
if not self._settings.load_balanced:
self._process_change(
ServerDescription(address, error=error))
if is_shutting_down or (err_ctx.max_wire_version <= 7):
# Clear the pool.
server.reset(service_id)
server.request_check()
elif not err_ctx.completed_handshake:
# Unknown command error during the connection handshake.
if not self._settings.load_balanced:
self._process_change(
ServerDescription(address, error=error))
# Clear the pool.
server.reset(service_id)
elif issubclass(exc_type, ConnectionFailure):
# "Client MUST replace the server's description with type Unknown
# ... MUST NOT request an immediate check of the server."
if not self._settings.load_balanced:
self._process_change(ServerDescription(address, error=error))
# Clear the pool.
server.reset(service_id)
# "When a client marks a server Unknown from `Network error when
# reading or writing`_, clients MUST cancel the isMaster check on
# that server and close the current monitoring connection."
server._monitor.cancel_check()
def _update_rs_from_primary(sds, replica_set_name, server_description,
max_election_id):
"""Update topology description from a primary's ismaster response.
Pass in a dict of ServerDescriptions, current replica set name, the
ServerDescription we are processing, and the TopologyDescription's
max_election_id if any.
Returns (new topology type, new replica_set_name, new max_election_id).
"""
if replica_set_name is None:
replica_set_name = server_description.replica_set_name
elif replica_set_name != server_description.replica_set_name:
# We found a primary but it doesn't have the replica_set_name
# provided by the user.
sds.pop(server_description.address)
return _check_has_primary(sds), replica_set_name, max_election_id
if server_description.election_id is not None:
if max_election_id and max_election_id > server_description.election_id:
# Stale primary, set to type Unknown.
address = server_description.address
sds[address] = ServerDescription(address)
return _check_has_primary(sds), replica_set_name, max_election_id
max_election_id = server_description.election_id
# We've heard from the primary. Is it the same primary as before?
for server in sds.values():
if (server.server_type is SERVER_TYPE.RSPrimary
and server.address != server_description.address):
# Reset old primary's type to Unknown.
sds[server.address] = ServerDescription(server.address)
# There can be only one prior primary.
break
# Discover new hosts from this primary's response.
for new_address in server_description.all_hosts:
if new_address not in sds:
sds[new_address] = ServerDescription(new_address)
# Remove hosts not in the response.
for addr in set(sds) - server_description.all_hosts:
sds.pop(addr)
# If the host list differs from the seed list, we may not have a primary
# after all.
return _check_has_primary(sds), replica_set_name, max_election_id
def _update_rs_no_primary_from_member(
sds,
replica_set_name,
server_description):
"""RS without known primary. Update from a non-primary's response.
Pass in a dict of ServerDescriptions, current replica set name, and the
ServerDescription we are processing.
Returns (new topology type, new replica_set_name).
"""
topology_type = TOPOLOGY_TYPE.ReplicaSetNoPrimary
if replica_set_name is None:
replica_set_name = server_description.replica_set_name
elif replica_set_name != server_description.replica_set_name:
sds.pop(server_description.address)
return topology_type, replica_set_name
# This isn't the primary's response, so don't remove any servers
# it doesn't report. Only add new servers.
for address in server_description.all_hosts:
if address not in sds:
sds[address] = ServerDescription(address)
if (server_description.me and
server_description.address != server_description.me):
sds.pop(server_description.address)
return topology_type, replica_set_name
def _run(self):
try:
prev_sd = self._server_description
try:
self._server_description = self._check_server()
except _OperationCancelled as exc:
_sanitize(exc)
# Already closed the connection, wait for the next check.
self._server_description = ServerDescription(
self._server_description.address, error=exc)
if prev_sd.is_server_type_known:
# Immediately retry since we've already waited 500ms to
# discover that we've been cancelled.
self._executor.skip_sleep()
return
# Update the Topology and clear the server pool on error.
self._topology.on_change(self._server_description,
reset_pool=self._server_description.error)
if (self._server_description.is_server_type_known
and self._server_description.topology_version):
self._start_rtt_monitor()
# Immediately check for the next streaming response.
self._executor.skip_sleep()
if self._server_description.error and prev_sd.is_server_type_known:
# Immediately retry on network errors.
self._executor.skip_sleep()
except ReferenceError:
# Topology was garbage-collected.
self.close()
def _ensure_opened(self):
"""Start monitors, or restart after a fork.
Hold the lock when calling this.
"""
if not self._opened:
self._opened = True
self._update_servers()
# Start or restart the events publishing thread.
if self._publish_tp or self._publish_server:
self.__events_executor.open()
# Start the SRV polling thread.
if self._srv_monitor and (self.description.topology_type
in SRV_POLLING_TOPOLOGIES):
self._srv_monitor.open()
if self._settings.load_balanced:
# Emit initial SDAM events for load balancer mode.
self._process_change(
ServerDescription(
self._seed_addresses[0],
IsMaster({
'ok': 1,
'serviceId': self._topology_id,
'maxWireVersion': 13
})))
# Ensure that the monitors are open.
for server in itervalues(self._servers):
server.open()
def _check_server(self):
"""Call hello or read the next streaming response.
Returns a ServerDescription.
"""
start = time.monotonic()
try:
try:
return self._check_once()
except (OperationFailure, NotPrimaryError) as exc:
# Update max cluster time even when hello fails.
details = cast(Mapping[str, Any], exc.details)
self._topology.receive_cluster_time(
details.get("$clusterTime"))
raise
except ReferenceError:
raise
except Exception as error:
_sanitize(error)
sd = self._server_description
address = sd.address
duration = time.monotonic() - start
if self._publish:
awaited = sd.is_server_type_known and sd.topology_version
self._listeners.publish_server_heartbeat_failed(
address, duration, error, awaited)
self._reset_connection()
if isinstance(error, _OperationCancelled):
raise
self._rtt_monitor.reset()
# Server type defaults to Unknown.
return ServerDescription(address, error=error)
def _check_with_retry(self):
"""Call ismaster once or twice. Reset server's pool on error.
Returns a ServerDescription.
"""
# According to the spec, if an ismaster call fails we reset the
# server's pool. If a server was once connected, change its type
# to Unknown only after retrying once.
address = self._server_description.address
retry = self._server_description.server_type != SERVER_TYPE.Unknown
try:
return self._check_once()
except ReferenceError:
raise
except Exception as error:
self._topology.reset_pool(address)
default = ServerDescription(address, error=error)
if not retry:
self._avg_round_trip_time.reset()
# Server type defaults to Unknown.
return default
# Try a second and final time. If it fails return original error.
try:
return self._check_once()
except ReferenceError:
raise
except Exception:
self._avg_round_trip_time.reset()
return default
def _updated_topology_description_srv_polling(topology_description, seedlist):
"""Return an updated copy of a TopologyDescription.
:Parameters:
- `topology_description`: the current TopologyDescription
- `seedlist`: a list of new seeds new ServerDescription that resulted from
an ismaster call
"""
# Create a copy of the server descriptions.
sds = topology_description.server_descriptions()
# If seeds haven't changed, don't do anything.
if set(sds.keys()) == set(seedlist):
return topology_description
# Add SDs corresponding to servers recently added to the SRV record.
for address in seedlist:
if address not in sds:
sds[address] = ServerDescription(address)
# Remove SDs corresponding to servers no longer part of the SRV record.
for address in list(sds.keys()):
if address not in seedlist:
sds.pop(address)
return TopologyDescription(
topology_description.topology_type,
sds,
topology_description.replica_set_name,
topology_description.max_set_version,
topology_description.max_election_id,
topology_description._topology_settings)
def handle_getlasterror(self, address, error_msg):
"""Clear our pool for a server, mark it Unknown, and check it soon."""
error = NotMasterError(error_msg, {'code': 10107, 'errmsg': error_msg})
with self._lock:
server = self._servers.get(address)
if server:
self._process_change(ServerDescription(address, error=error),
True)
server.request_check()
def _handle_error(self, address, err_ctx):
if self._is_stale_error(address, err_ctx):
return
server = self._servers[address]
error = err_ctx.error
exc_type = type(error)
if issubclass(exc_type, NetworkTimeout):
# The socket has been closed. Don't reset the server.
# Server Discovery And Monitoring Spec: "When an application
# operation fails because of any network error besides a socket
# timeout...."
return
elif issubclass(exc_type, NotMasterError):
# As per the SDAM spec if:
# - the server sees a "not master" error, and
# - the server is not shutting down, and
# - the server version is >= 4.2, then
# we keep the existing connection pool, but mark the server type
# as Unknown and request an immediate check of the server.
# Otherwise, we clear the connection pool, mark the server as
# Unknown and request an immediate check of the server.
err_code = error.details.get('code', -1)
is_shutting_down = err_code in helpers._SHUTDOWN_CODES
# Mark server Unknown, clear the pool, and request check.
self._process_change(ServerDescription(address, error=error))
if is_shutting_down or (err_ctx.max_wire_version <= 7):
# Clear the pool.
server.reset()
server.request_check()
elif issubclass(exc_type, ConnectionFailure):
# "Client MUST replace the server's description with type Unknown
# ... MUST NOT request an immediate check of the server."
self._process_change(ServerDescription(address, error=error))
# Clear the pool.
server.reset()
elif issubclass(exc_type, OperationFailure):
# Do not request an immediate check since the server is likely
# shutting down.
if error.code in helpers._NOT_MASTER_CODES:
self._process_change(ServerDescription(address, error=error))
# Clear the pool.
server.reset()
def _run_scenario(self):
class NoopMonitor(Monitor):
"""Override the _run method to do nothing."""
def _run(self):
time.sleep(0.05)
m = single_client(h=scenario_def['uri'],
p=27017,
event_listeners=[self.all_listener],
_monitor_class=NoopMonitor)
topology = m._get_topology()
try:
for phase in scenario_def['phases']:
for (source, response) in phase['responses']:
source_address = clean_node(source)
topology.on_change(
ServerDescription(address=source_address,
ismaster=IsMaster(response),
round_trip_time=0))
expected_results = phase['outcome']['events']
expected_len = len(expected_results)
wait_until(
lambda: len(self.all_listener.results) >= expected_len,
"publish all events",
timeout=15)
# Wait some time to catch possible lagging extra events.
time.sleep(0.5)
i = 0
while i < expected_len:
result = self.all_listener.results[i] if len(
self.all_listener.results) > i else None
# The order of ServerOpening/ClosedEvents doesn't matter
if isinstance(result, (monitoring.ServerOpeningEvent,
monitoring.ServerClosedEvent)):
i, passed, message = compare_multiple_events(
i, expected_results, self.all_listener.results)
self.assertTrue(passed, message)
else:
self.assertTrue(
*compare_events(expected_results[i], result))
i += 1
# Assert no extra events.
extra_events = self.all_listener.results[expected_len:]
if extra_events:
self.fail('Extra events %r' % (extra_events, ))
self.all_listener.reset()
finally:
m.close()
def reset(self):
"""A copy of this description, with all servers marked Unknown."""
if self._topology_type == TOPOLOGY_TYPE.ReplicaSetWithPrimary:
topology_type = TOPOLOGY_TYPE.ReplicaSetNoPrimary
else:
topology_type = self._topology_type
# The default ServerDescription's type is Unknown.
sds = dict((address, ServerDescription(address))
for address in self._server_descriptions)
return TopologyDescription(topology_type, sds, self._replica_set_name)
def make_server_description(server, hosts):
"""Make a ServerDescription from server info in a JSON test."""
server_type = server['type']
if server_type in ("Unknown", "PossiblePrimary"):
return ServerDescription(clean_node(server['address']), Hello({}))
hello_response = {'ok': True, 'hosts': hosts}
if server_type not in ("Standalone", "Mongos", "RSGhost"):
hello_response['setName'] = "rs"
if server_type == "RSPrimary":
hello_response[HelloCompat.LEGACY_CMD] = True
elif server_type == "RSSecondary":
hello_response['secondary'] = True
elif server_type == "Mongos":
hello_response['msg'] = 'isdbgrid'
elif server_type == "RSGhost":
hello_response['isreplicaset'] = True
elif server_type == "RSArbiter":
hello_response['arbiterOnly'] = True
hello_response['lastWrite'] = {
'lastWriteDate': make_last_write_date(server)
}
for field in 'maxWireVersion', 'tags', 'idleWritePeriodMillis':
if field in server:
hello_response[field] = server[field]
hello_response.setdefault('maxWireVersion', 6)
# Sets _last_update_time to now.
sd = ServerDescription(clean_node(server['address']),
Hello(hello_response),
round_trip_time=server['avg_rtt_ms'] / 1000.0)
if 'lastUpdateTime' in server:
sd._last_update_time = server['lastUpdateTime'] / 1000.0 # ms to sec.
return sd
def _check_with_retry(self):
"""Call ismaster once or twice. Reset server's pool on error.
Returns a ServerDescription.
"""
# According to the spec, if an ismaster call fails we reset the
# server's pool. If a server was once connected, change its type
# to Unknown only after retrying once.
address = self._server_description.address
retry = True
metadata = None
if self._server_description.server_type == SERVER_TYPE.Unknown:
retry = False
metadata = self._pool.opts.metadata
start = _time()
try:
cluster_time = self._topology.max_cluster_time()
# If the server type is unknown, send metadata with first check.
return self._check_once(metadata=metadata,
cluster_time=cluster_time)
except ReferenceError:
raise
except Exception as error:
error_time = _time() - start
if self._publish:
self._listeners.publish_server_heartbeat_failed(
address, error_time, error)
self._topology.reset_pool(address)
default = ServerDescription(address, error=error)
if not retry:
self._avg_round_trip_time.reset()
# Server type defaults to Unknown.
return default
# Try a second and final time. If it fails return original error.
# Always send metadata: this is a new connection.
start = _time()
try:
cluster_time = self._topology.max_cluster_time()
return self._check_once(metadata=self._pool.opts.metadata,
cluster_time=cluster_time)
except ReferenceError:
raise
except Exception as error:
error_time = _time() - start
if self._publish:
self._listeners.publish_server_heartbeat_failed(
address, error_time, error)
self._avg_round_trip_time.reset()
return default
def _reset_server(self, address, reset_pool, error):
"""Mark a server Unknown and optionally reset it's pool.
Hold the lock when calling this. Does *not* request an immediate check.
"""
server = self._servers.get(address)
# "server" is None if another thread removed it from the topology.
if server:
if reset_pool:
server.reset()
# Mark this server Unknown.
self._process_change(ServerDescription(address, error=error))
def _check_once(self):
"""A single attempt to call ismaster.
Returns a ServerDescription, or raises an exception.
"""
with self._pool.get_socket({}) as sock_info:
response, round_trip_time = self._check_with_socket(sock_info)
self._avg_round_trip_time.add_sample(round_trip_time)
sd = ServerDescription(
address=self._server_description.address,
ismaster=response,
round_trip_time=self._avg_round_trip_time.get())
return sd
def _run(self):
try:
if self._server_description.address != ('a', 27017):
# Because PyMongo doesn't keep information about
# the order of addresses, we might accidentally
# start a MockMonitor on the wrong server first,
# so we need to only mock responses for the server
# the test's response is supposed to come from.
return
response = next(responses)[1]
isMaster = IsMaster(response)
self._server_description = ServerDescription(
address=self._server_description.address,
ismaster=isMaster)
self._topology.on_change(self._server_description)
except (ReferenceError, StopIteration):
# Topology was garbage-collected.
self.close()
def _check_once(self):
"""A single attempt to call ismaster.
Returns a ServerDescription, or raises an exception.
"""
address = self._server_description.address
if self._publish:
self._listeners.publish_server_heartbeat_started(address)
with self._pool.get_socket({}) as sock_info:
response, round_trip_time = self._check_with_socket(sock_info)
self._avg_round_trip_time.add_sample(round_trip_time)
sd = ServerDescription(
address=address,
ismaster=response,
round_trip_time=self._avg_round_trip_time.get())
if self._publish:
self._listeners.publish_server_heartbeat_succeeded(
address, round_trip_time, response)
return sd
def make_server_description(server, hosts):
"""Make ServerDescription from server info from JSON file."""
ismaster_response = {}
ismaster_response['tags'] = server['tags']
ismaster_response['ok'] = True
ismaster_response['hosts'] = hosts
server_type = server['type']
if server_type != "Standalone" and server_type != "Mongos":
ismaster_response['setName'] = True
if server_type == "RSPrimary":
ismaster_response['ismaster'] = True
elif server_type == "RSSecondary":
ismaster_response['secondary'] = True
elif server_type == "Mongos":
ismaster_response['msg'] = 'isdbgrid'
return ServerDescription(clean_node(server['address']),
IsMaster(ismaster_response),
round_trip_time=server['avg_rtt_ms'])
def _updated_topology_description_srv_polling(topology_description, seedlist):
"""Return an updated copy of a TopologyDescription.
:Parameters:
- `topology_description`: the current TopologyDescription
- `seedlist`: a list of new seeds new ServerDescription that resulted from
a hello call
"""
# Create a copy of the server descriptions.
sds = topology_description.server_descriptions()
# If seeds haven't changed, don't do anything.
if set(sds.keys()) == set(seedlist):
return topology_description
# Remove SDs corresponding to servers no longer part of the SRV record.
for address in list(sds.keys()):
if address not in seedlist:
sds.pop(address)
if topology_description.srv_max_hosts != 0:
new_hosts = set(seedlist) - set(sds.keys())
n_to_add = topology_description.srv_max_hosts - len(sds)
if n_to_add > 0:
seedlist = sample(new_hosts, min(n_to_add, len(new_hosts)))
else:
seedlist = []
# Add SDs corresponding to servers recently added to the SRV record.
for address in seedlist:
if address not in sds:
sds[address] = ServerDescription(address)
return TopologyDescription(
topology_description.topology_type,
sds,
topology_description.replica_set_name,
topology_description.max_set_version,
topology_description.max_election_id,
topology_description._topology_settings,
)
def _check_once(self):
"""A single attempt to call hello.
Returns a ServerDescription, or raises an exception.
"""
address = self._server_description.address
if self._publish:
self._listeners.publish_server_heartbeat_started(address)
if self._cancel_context and self._cancel_context.cancelled:
self._reset_connection()
with self._pool.get_socket() as sock_info:
self._cancel_context = sock_info.cancel_context
response, round_trip_time = self._check_with_socket(sock_info)
if not response.awaitable:
self._rtt_monitor.add_sample(round_trip_time)
sd = ServerDescription(address, response,
self._rtt_monitor.average())
if self._publish:
self._listeners.publish_server_heartbeat_succeeded(
address, round_trip_time, response, response.awaitable)
return sd
def _ensure_opened(self):
"""Start monitors, or restart after a fork.
Hold the lock when calling this.
"""
if self._closed:
raise InvalidOperation("Cannot use MongoClient after close")
if not self._opened:
self._opened = True
self._update_servers()
# Start or restart the events publishing thread.
if self._publish_tp or self._publish_server:
self.__events_executor.open()
# Start the SRV polling thread.
if self._srv_monitor and (self.description.topology_type
in SRV_POLLING_TOPOLOGIES):
self._srv_monitor.open()
if self._settings.load_balanced:
# Emit initial SDAM events for load balancer mode.
self._process_change(
ServerDescription(
self._seed_addresses[0],
Hello({
"ok": 1,
"serviceId": self._topology_id,
"maxWireVersion": 13
}),
))
# Ensure that the monitors are open.
for server in self._servers.values():
server.open()
def got_ismaster(topology, server_address, ismaster_response):
server_description = ServerDescription(server_address,
IsMaster(ismaster_response), 0)
topology.on_change(server_description)
def _check_once(self, metadata=None, cluster_time=None):
address = self._server_description.address
response, rtt = self.client.mock_is_master('%s:%d' % address)
return ServerDescription(address, IsMaster(response), rtt)
def get_server_descriptions(self):
"""Initial dict of (address, ServerDescription) for all seeds."""
return dict([(address, ServerDescription(address))
for address in self.seeds])
def test_unknown(self):
# Default, no ismaster_response.
s = ServerDescription(address)
self.assertEqual(SERVER_TYPE.Unknown, s.server_type)
self.assertFalse(s.is_writable)
self.assertFalse(s.is_readable)
def parse_ismaster_response(doc):
ismaster_response = IsMaster(doc)
return ServerDescription(address, ismaster_response)
def disconnected(topology, server_address):
# Create new description of server type Unknown.
topology.on_change(ServerDescription(server_address))
def got_hello(topology, server_address, hello_response):
server_description = ServerDescription(
server_address, Hello(hello_response), 0)
topology.on_change(server_description)
