def __check_response_to_last_error(self, response):
"""Check a response to a lastError message for errors.
`response` is a byte string representing a response to the message.
If it represents an error response we raise OperationFailure.
Return the response as a document.
"""
response = helpers._unpack_response(response)
assert response["number_returned"] == 1
error = response["data"][0]
helpers._check_command_response(error, self.disconnect)
error_msg = error.get("err", "")
if error_msg is None:
return error
if error_msg.startswith("not master"):
self.disconnect()
raise AutoReconnect(error_msg)
if "code" in error:
if error["code"] in [11000, 11001, 12582]:
raise DuplicateKeyError(error["err"])
else:
raise OperationFailure(error["err"], error["code"])
else:
raise OperationFailure(error["err"])
def _check_with_socket(self, *args, **kwargs):
ismaster_count[0] += 1
if ismaster_count[0] in (1, 3):
return IsMaster({'ok': 1, 'maxWireVersion': 6}), 0
else:
raise AutoReconnect('mock monitor error #%s' %
(ismaster_count[0], ))
def _send_message(self, message, with_last_error=False):
"""Say something to Mongo.
Raises ConnectionFailure if the message cannot be sent. Raises
OperationFailure if `with_last_error` is ``True`` and the
response to the getLastError call returns an error. Return the
response from lastError, or ``None`` if `with_last_error`
is ``False``.
:Parameters:
- `message`: message to send
- `with_last_error`: check getLastError status after sending the
message
"""
sock = self.__pool.socket()
try:
(request_id, data) = message
sock.sendall(data)
# Safe mode. We pack the message together with a lastError
# message and send both. We then get the response (to the
# lastError) and raise OperationFailure if it is an error
# response.
if with_last_error:
response = self.__receive_message_on_socket(
1, request_id, sock)
return self.__check_response_to_last_error(response)
return None
except (ConnectionFailure, socket.error), e:
self._reset()
raise AutoReconnect(str(e))
def __socket(self):
"""Get a socket from the pool.
If it's been > 1 second since the last time we checked out a
socket, we also check to see if the socket has been closed -
this let's us avoid seeing *some*
:class:`~pymongo.errors.AutoReconnect` exceptions on server
hiccups, etc. We only do this if it's been > 1 second since
the last socket checkout, to keep performance reasonable - we
can't avoid those completely anyway.
"""
host, port = (self.__host, self.__port)
if host is None or port is None:
host, port = self.__find_node()
try:
sock = self.__pool.get_socket(host, port)
except socket.error:
self.disconnect()
raise AutoReconnect("could not connect to %s:%d" % (host, port))
t = time.time()
if t - self.__last_checkout > 1:
if _closed(sock):
self.disconnect()
sock = self.__pool.get_socket(host, port)
self.__last_checkout = t
return sock
def _send_message(self, msg, safe=False, _connection_to_use=None):
"""Say something to Mongo.
Raises ConnectionFailure if the message cannot be sent. Raises
OperationFailure if `safe` is ``True`` and the
response to the getLastError call returns an error. Return the
response from lastError, or ``None`` if `safe` is ``False``.
:Parameters:
- `msg`: message to send
- `safe`: check getLastError status after sending the message
"""
if _connection_to_use in (None, -1):
mongo = self.__find_primary()
else:
mongo = self.__pools[_connection_to_use]
try:
sock = self.__socket(mongo)
rqst_id, data = self.__check_bson_size(msg, mongo['max_bson_size'])
sock.sendall(data)
# Safe mode. We pack the message together with a lastError
# message and send both. We then get the response (to the
# lastError) and raise OperationFailure if it is an error
# response.
if safe:
response = self.__recv_msg(1, rqst_id, sock)
return self.__check_response_to_last_error(response)
return None
except (ConnectionFailure, socket.error), why:
mongo['pool'].discard_socket()
if _connection_to_use in (None, -1):
self.disconnect()
raise AutoReconnect(str(why))
def __find_master(self):
"""Create a new socket and use it to figure out who the master is.
Sets __host and __port so that :attr:`host` and :attr:`port`
will return the address of the master. Also (possibly) updates
any replSet information.
"""
# Special case the first node to try to get the primary or any
# additional hosts from a replSet:
first = iter(self.__nodes).next()
primary = self.__try_node(first)
if primary is True:
return first
# no network error
if self.__slave_okay and primary is not None:
return first
# Wasn't the first node, but we got a primary - let's try it:
tried = [first]
if primary:
if self.__try_node(primary) is True:
return primary
tried.append(primary)
nodes = self.__nodes - set(tried)
# Just scan
# TODO parallelize these to minimize connect time?
for node in nodes:
if self.__try_node(node) is True:
return node
raise AutoReconnect("could not find master/primary")
def __socket(self):
"""Get a SocketInfo from the pool.
"""
host, port = (self.__host, self.__port)
if host is None or (port is None and '/' not in host):
host, port = self.__find_node()
try:
if self.auto_start_request and not self.in_request():
self.start_request()
sock_info = self.__pool.get_socket((host, port))
except socket.error as why:
self.disconnect()
# Check if a unix domain socket
if host.endswith('.sock'):
host_details = "%s:" % host
else:
host_details = "%s:%d:" % (host, port)
raise AutoReconnect("could not connect to "
"%s %s" % (host_details, str(why)))
try:
self.__check_auth(sock_info)
except OperationFailure:
self.__pool.maybe_return_socket(sock_info)
raise
return sock_info
def got_app_error(topology, app_error):
server_address = common.partition_node(app_error['address'])
server = topology.get_server_by_address(server_address)
error_type = app_error['type']
generation = app_error.get(
'generation', server.pool.gen.get_overall())
when = app_error['when']
max_wire_version = app_error['maxWireVersion']
# XXX: We could get better test coverage by mocking the errors on the
# Pool/SocketInfo.
try:
if error_type == 'command':
_check_command_response(app_error['response'], max_wire_version)
_check_write_command_response(app_error['response'])
elif error_type == 'network':
raise AutoReconnect('mock non-timeout network error')
elif error_type == 'timeout':
raise NetworkTimeout('mock network timeout error')
else:
raise AssertionError('unknown error type: %s' % (error_type,))
assert False
except (AutoReconnect, NotPrimaryError, OperationFailure) as e:
if when == 'beforeHandshakeCompletes':
completed_handshake = False
elif when == 'afterHandshakeCompletes':
completed_handshake = True
else:
assert False, 'Unknown when field %s' % (when,)
topology.handle_error(
server_address, _ErrorContext(e, max_wire_version, generation,
completed_handshake, None))
def get_entries(self, find=None, sort=None, skip=None, limit=None):
"""Getting logentry data
Args:
find: dict, pymongo-find clause for filtering documents
sort: single key or a list of (key, direction) pairs specifying the keys to sort on
limit: int, the number of results to be returned
Returns:
pymongo cursor
"""
if sort is None:
sort = [('datetimestamp', DESCENDING)]
if limit is None:
limit = 100
if skip is None:
skip = 0
if find is not None and isinstance(find, dict):
find = self.__prepare_find(find)
try:
cursor = db[collection].find(find).sort(sort).skip(skip).limit(
limit)
except TypeError, e:
error = u'Неверный тип параметров ({0})'.format(e)
raise TypeError(error)
except AutoReconnect, e:
error = u'Потеряно подключение к БД ({0})'.format(e)
raise AutoReconnect(error)
def _unpack_response(response, cursor_id=None, as_class=dict, tz_aware=False):
"""Unpack a response from the database.
Check the response for errors and unpack, returning a dictionary
containing the response data.
:Parameters:
- `response`: byte string as returned from the database
- `cursor_id` (optional): cursor_id we sent to get this response -
used for raising an informative exception when we get cursor id not
valid at server response
- `as_class` (optional): class to use for resulting documents
"""
response_flag = struct.unpack("<i", response[:4])[0]
if response_flag & 1:
# Shouldn't get this response if we aren't doing a getMore
assert cursor_id is not None
raise OperationFailure("cursor id '%s' not valid at server" %
cursor_id)
elif response_flag & 2:
error_object = bson.BSON(response[20:]).decode()
if error_object["$err"].startswith("not master"):
raise AutoReconnect("master has changed")
raise OperationFailure("database error: %s" %
error_object["$err"])
result = {}
result["cursor_id"] = struct.unpack("<q", response[4:12])[0]
result["starting_from"] = struct.unpack("<i", response[12:16])[0]
result["number_returned"] = struct.unpack("<i", response[16:20])[0]
result["data"] = bson.decode_all(response[20:], as_class, tz_aware)
assert len(result["data"]) == result["number_returned"]
return result
def _send_message_with_response(self,
message,
_must_use_master=False,
**kwargs):
"""Send a message to Mongo and return the response.
Sends the given message and returns the response.
:Parameters:
- `message`: (request_id, data) pair making up the message to send
"""
sock_info = self.__socket()
try:
try:
if "network_timeout" in kwargs:
sock_info.sock.settimeout(kwargs["network_timeout"])
return self.__send_and_receive(message, sock_info)
except (ConnectionFailure, socket.error), e:
self.disconnect()
raise AutoReconnect(str(e))
finally:
if "network_timeout" in kwargs:
try:
# Restore the socket's original timeout and return it to
# the pool
sock_info.sock.settimeout(self.__net_timeout)
self.__pool.maybe_return_socket(sock_info)
except socket.error:
# There was an exception and we've closed the socket
pass
else:
self.__pool.maybe_return_socket(sock_info)
def __find_node(self):
"""Find a host, port pair suitable for our connection type.
If only one host was supplied to __init__ see if we can connect
to it. Don't check if the host is a master/primary so we can make
a direct connection to read from a slave.
If more than one host was supplied treat them as a seed list for
connecting to a replica set. Try to find the primary and fail if
we can't. Possibly updates any replSet information on success.
If the list of hosts is not a seed list for a replica set the
behavior is still the same. We iterate through the list trying
to find a host we can send write operations to.
In either case a connection to an arbiter will never succeed.
Sets __host and __port so that :attr:`host` and :attr:`port`
will return the address of the connected host.
"""
for candidate in self.__nodes:
node = self.__try_node(candidate)
if node:
return node
self.disconnect()
raise AutoReconnect("could not find master/primary")
def _check_command_response(response, reset, msg="%s", allowable_errors=[]):
if not response["ok"]:
if "wtimeout" in response and response["wtimeout"]:
raise TimeoutError(msg % response["errmsg"])
details = response
# Mongos returns the error details in a 'raw' object
# for some errors.
if "raw" in response:
for shard in response["raw"].values():
if not shard.get("ok"):
# Just grab the first error...
details = shard
break
if not details["errmsg"] in allowable_errors:
if details["errmsg"] == "not master":
if reset is not None:
reset()
raise AutoReconnect("not master")
if details["errmsg"] == "db assertion failure":
ex_msg = ("db assertion failure, assertion: '%s'" %
details.get("assertion", ""))
if "assertionCode" in details:
ex_msg += (", assertionCode: %d" %
(details["assertionCode"], ))
raise OperationFailure(ex_msg, details.get("assertionCode"))
raise OperationFailure(msg % details["errmsg"])
def got_app_error(topology, app_error):
server_address = common.partition_node(app_error['address'])
server = topology.get_server_by_address(server_address)
error_type = app_error['type']
generation = app_error.get('generation', server.pool.generation)
when = app_error['when']
max_wire_version = app_error['maxWireVersion']
# XXX: We could get better test coverage by mocking the errors on the
# Pool/SocketInfo.
try:
if error_type == 'command':
_check_command_response(app_error['response'])
elif error_type == 'network':
raise AutoReconnect('mock non-timeout network error')
elif error_type == 'timeout':
raise NetworkTimeout('mock network timeout error')
else:
raise AssertionError('unknown error type: %s' % (error_type, ))
assert False
except (AutoReconnect, NotMasterError, OperationFailure) as e:
if when == 'beforeHandshakeCompletes' and error_type == 'timeout':
raise unittest.SkipTest('PYTHON-2211')
topology.handle_error(server_address,
_ErrorContext(e, max_wire_version, generation))
def modify_text(user,
oref,
versionTitle,
language,
text,
versionSource,
tries=0):
try:
tracker.modify_text(user,
oref,
versionTitle,
language,
text,
versionSource,
method="API",
skip_links=False)
except UnicodeEncodeError:
print("UnicodeEncodeError: {}".format(oref.normal()))
pass # there seems to be unicode error in "/app/sefaria/system/varnish/wrapper.py", line 55
except AutoReconnect:
if tries < 200:
modify_text(user,
oref,
versionTitle,
language,
text,
versionSource,
tries=tries + 1)
else:
raise AutoReconnect(
"Tried so hard but got so many autoreconnects...")
except AssertionError:
pass
def __find_node(self, seeds=None):
"""Find a host, port pair suitable for our connection type.
If only one host was supplied to __init__ see if we can connect
to it. Don't check if the host is a master/primary so we can make
a direct connection to read from a secondary or send commands to
an arbiter.
If more than one host was supplied treat them as a seed list for
connecting to a replica set or to support high availability for
mongos. If connecting to a replica set try to find the primary
and fail if we can't, possibly updating any replSet information
on success. If a mongos seed list was provided find the "nearest"
mongos and return it.
Otherwise we iterate through the list trying to find a host we can
send write operations to.
Sets __host and __port so that :attr:`host` and :attr:`port`
will return the address of the connected host. Sets __is_primary to
True if this is a primary or master, else False. Sets __is_mongos
to True if the connection is to a mongos.
"""
errors = []
mongos_candidates = []
candidates = seeds or self.__nodes.copy()
for candidate in candidates:
try:
node, ismaster, isdbgrid, res_time = self.__try_node(candidate)
self.__is_primary = ismaster
self.__is_mongos = isdbgrid
# No need to calculate nearest if we only have one mongos.
if isdbgrid and not self.__direct:
mongos_candidates.append((node, res_time))
continue
elif len(mongos_candidates):
raise ConfigurationError("Seed list cannot contain a mix "
"of mongod and mongos instances.")
return node
except Exception as why:
errors.append(str(why))
# If we have a mongos seed list, pick the "nearest" member.
if len(mongos_candidates):
self.__is_mongos = True
return self.__pick_nearest(mongos_candidates)
# Otherwise, try any hosts we discovered that were not in the seed list.
for candidate in self.__nodes - candidates:
try:
node, ismaster, isdbgrid, _ = self.__try_node(candidate)
self.__is_primary = ismaster
self.__is_mongos = isdbgrid
return node
except Exception as why:
errors.append(str(why))
# Couldn't find a suitable host.
self.disconnect()
raise AutoReconnect(', '.join(errors))
def foo(self):
if retries[0] == 2:
retries[0] -= 1
raise OperationFailure('error')
elif retries[0] == 1:
retries[0] -= 1
raise AutoReconnect('error')
return "success"
def test_ensure_indexes_other_error(self):
# same as above, but no swallowing
collection = self.MockSession.db[self.MyDoc.__mongometa__.name]
ensure_index = collection.ensure_index
ensure_index.side_effect = AutoReconnect('blah blah')
self.assertRaises(AutoReconnect, lambda: self.MyDoc.m)
assert ensure_index.called
def _check_with_socket(self, sock_info, metadata=None):
if available:
return (IsMaster({
'ok': 1,
'maxWireVersion': 6
}), round_trip_time)
else:
raise AutoReconnect('mock monitor error')
def _check_with_socket(self, *args, **kwargs):
if available:
return (IsMaster({
'ok': 1,
'maxWireVersion': 6
}), round_trip_time)
else:
raise AutoReconnect('mock monitor error')
def _raise_connection_failure(address, error):
"""Convert a socket.error to ConnectionFailure and raise it."""
host, port = address
msg = '%s:%d: %s' % (host, port, error)
if isinstance(error, socket.timeout):
raise NetworkTimeout(msg)
else:
raise AutoReconnect(msg)
def _check_command_response(response, reset, msg=None, allowable_errors=None):
"""Check the response to a command for errors.
"""
if "ok" not in response:
# Server didn't recognize our message as a command.
raise OperationFailure(response.get("$err"), response.get("code"),
response)
if response.get("wtimeout", False):
# MongoDB versions before 1.8.0 return the error message in an "errmsg"
# field. If "errmsg" exists "err" will also exist set to None, so we
# have to check for "errmsg" first.
raise WTimeoutError(response.get("errmsg", response.get("err")),
response.get("code"), response)
if not response["ok"]:
details = response
# Mongos returns the error details in a 'raw' object
# for some errors.
if "raw" in response:
for shard in response["raw"].itervalues():
if not shard.get("ok"):
# Just grab the first error...
details = shard
break
errmsg = details["errmsg"]
if allowable_errors is None or errmsg not in allowable_errors:
# Server is "not master" or "recovering"
if (errmsg.startswith("not master")
or errmsg.startswith("node is recovering")):
if reset is not None:
reset()
raise AutoReconnect(errmsg)
# Server assertion failures
if errmsg == "db assertion failure":
errmsg = ("db assertion failure, assertion: '%s'" %
details.get("assertion", ""))
raise OperationFailure(errmsg, details.get("assertionCode"),
response)
# Other errors
code = details.get("code")
# findAndModify with upsert can raise duplicate key error
if code in (11000, 11001, 12582):
raise DuplicateKeyError(errmsg, code, response)
elif code == 50:
raise ExecutionTimeout(errmsg, code, response)
# wtimeout from write commands
if "errInfo" in details and details["errInfo"].get("wtimeout"):
raise WTimeoutError(errmsg, code, response)
msg = msg or "%s"
raise OperationFailure(msg % errmsg, code, response)
def _send_message(self,
message,
with_last_error=False,
check_primary=True):
"""Say something to Mongo.
Raises ConnectionFailure if the message cannot be sent. Raises
OperationFailure if `with_last_error` is ``True`` and the
response to the getLastError call returns an error. Return the
response from lastError, or ``None`` if `with_last_error`
is ``False``.
:Parameters:
- `message`: message to send
- `with_last_error`: check getLastError status after sending the
message
- `check_primary`: don't try to write to a non-primary; see
kill_cursors for an exception to this rule
"""
if check_primary and not with_last_error and not self.is_primary:
# The write won't succeed, bail as if we'd done a getLastError
raise AutoReconnect("not master")
sock_info = self.__socket()
try:
(request_id, data) = self.__check_bson_size(message)
sock_info.sock.sendall(data)
# Safe mode. We pack the message together with a lastError
# message and send both. We then get the response (to the
# lastError) and raise OperationFailure if it is an error
# response.
rv = None
if with_last_error:
response = self.__receive_message_on_socket(
1, request_id, sock_info)
rv = self.__check_response_to_last_error(response)
self.__pool.maybe_return_socket(sock_info)
return rv
except OperationFailure:
self.__pool.maybe_return_socket(sock_info)
raise
except (ConnectionFailure, socket.error), e:
self.disconnect()
raise AutoReconnect(str(e))
def mock_is_master(self, host):
"""Return mock ismaster response (a dict) and round trip time."""
if host in self.mock_wire_versions:
min_wire_version, max_wire_version = self.mock_wire_versions[host]
else:
min_wire_version = common.MIN_SUPPORTED_WIRE_VERSION
max_wire_version = common.MAX_SUPPORTED_WIRE_VERSION
max_write_batch_size = self.mock_max_write_batch_sizes.get(
host, common.MAX_WRITE_BATCH_SIZE)
rtt = self.mock_rtts.get(host, 0)
# host is like 'a:1'.
if host in self.mock_down_hosts:
raise NetworkTimeout('mock timeout')
elif host in self.mock_standalones:
response = {
'ok': 1,
'ismaster': True,
'minWireVersion': min_wire_version,
'maxWireVersion': max_wire_version,
'maxWriteBatchSize': max_write_batch_size
}
elif host in self.mock_members:
ismaster = (host == self.mock_primary)
# Simulate a replica set member.
response = {
'ok': 1,
'ismaster': ismaster,
'secondary': not ismaster,
'setName': 'rs',
'hosts': self.mock_ismaster_hosts,
'minWireVersion': min_wire_version,
'maxWireVersion': max_wire_version,
'maxWriteBatchSize': max_write_batch_size
}
if self.mock_primary:
response['primary'] = self.mock_primary
elif host in self.mock_mongoses:
response = {
'ok': 1,
'ismaster': True,
'minWireVersion': min_wire_version,
'maxWireVersion': max_wire_version,
'msg': 'isdbgrid',
'maxWriteBatchSize': max_write_batch_size
}
else:
# In test_internal_ips(), we try to connect to a host listed
# in ismaster['hosts'] but not publicly accessible.
raise AutoReconnect('Unknown host: %s' % host)
return response, rtt
def foo(self):
if retries[0] == 2:
retries[0] -= 1
raise OperationFailure('error')
elif retries[0] == 1:
register_log_exception_hook(hook2)
retries[0] -= 1
raise AutoReconnect('error')
return "success"
def test_handle_error_removed_server(self):
t = create_mock_topology(replica_set_name='rs')
# No error resetting a server not in the TopologyDescription.
errctx = _ErrorContext(AutoReconnect('mock'), 0, 0, True)
t.handle_error(('b', 27017), errctx)
# Server was *not* added as type Unknown.
self.assertFalse(t.has_server(('b', 27017)))
def _connect(self):
"""Connecting to mongo database."""
try:
self.connection = Connection(host=self.host, port=self.port)
except AutoReconnect, e:
if self.fail_silently:
return
else:
raise AutoReconnect(e)
def __try_node(self, node):
"""Try to connect to this node and see if it works for our connection
type. Returns ((host, port), ismaster, isdbgrid, res_time).
:Parameters:
- `node`: The (host, port) pair to try.
"""
self.disconnect()
self.__host, self.__port = node
# Call 'ismaster' directly so we can get a response time.
sock_info = self.__socket()
response, res_time = self.__simple_command(sock_info,
'admin',
{'ismaster': 1})
self.__pool.maybe_return_socket(sock_info)
# Are we talking to a mongos?
isdbgrid = response.get('msg', '') == 'isdbgrid'
if "maxBsonObjectSize" in response:
self.__max_bson_size = response["maxBsonObjectSize"]
# Replica Set?
if not self.__direct:
# Check that this host is part of the given replica set.
if self.__repl:
set_name = response.get('setName')
# The 'setName' field isn't returned by mongod before 1.6.2
# so we can't assume that if it's missing this host isn't in
# the specified set.
if set_name and set_name != self.__repl:
raise ConfigurationError("%s:%d is not a member of "
"replica set %s"
% (node[0], node[1], self.__repl))
if "hosts" in response:
self.__nodes = set([_partition_node(h)
for h in response["hosts"]])
else:
# The user passed a seed list of standalone or
# mongos instances.
self.__nodes.add(node)
if response["ismaster"]:
return node, True, isdbgrid, res_time
elif "primary" in response:
candidate = _partition_node(response["primary"])
return self.__try_node(candidate)
# Explain why we aren't using this connection.
raise AutoReconnect('%s:%d is not primary or master' % node)
# Direct connection
if response.get("arbiterOnly", False) and not self.__direct:
raise ConfigurationError("%s:%d is an arbiter" % node)
return node, response['ismaster'], isdbgrid, res_time
def test_ensure_indexes_slave(self):
# on a slave, an error will be thrown, but it should be swallowed
collection = self.MockSession.db[self.MyDoc.__mongometa__.name]
ensure_index = collection.ensure_index
ensure_index.side_effect = AutoReconnect('not master')
self.MyDoc.m
assert ensure_index.called
# don't keep trying after it failed once
self.MyDoc.m
assert ensure_index.call_count == 1, ensure_index.call_args_list
def _receive_data_on_socket(sock, length):
msg = b""
while length:
chunk = sock.recv(length)
if chunk == b"":
raise AutoReconnect("connection closed")
length -= len(chunk)
msg += chunk
return msg
