def _first_batch(sock_info, namespace, query, ntoreturn, slave_ok, codec_options, read_preference, cmd, listeners):
"""Simple query helper for retrieving a first (and possibly only) batch."""
query = _Query(0, namespace, 0, ntoreturn, query, None, codec_options, read_preference, 0, ntoreturn)
name = next(iter(cmd))
duration = None
publish = listeners.enabled_for_commands
if publish:
start = datetime.datetime.now()
request_id, msg, max_doc_size = query.get_message(slave_ok, sock_info.is_mongos)
if publish:
encoding_duration = datetime.datetime.now() - start
listeners.publish_command_start(cmd, namespace.split(".", 1)[0], request_id, sock_info.address)
start = datetime.datetime.now()
sock_info.send_message(msg, max_doc_size)
response = sock_info.receive_message(1, request_id)
try:
result = _unpack_response(response, None, codec_options)
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
if isinstance(exc, (NotMasterError, OperationFailure)):
failure = exc.details
else:
failure = _convert_exception(exc)
listeners.publish_command_failure(duration, failure, name, request_id, sock_info.address)
raise
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
listeners.publish_command_success(duration, result, name, request_id, sock_info.address)
return result
def _first_batch(sock_info, namespace, query, ntoreturn, slave_ok,
codec_options, read_preference, cmd, listeners):
"""Simple query helper for retrieving a first (and possibly only) batch."""
query = _Query(0, namespace, 0, ntoreturn, query, None, codec_options,
read_preference, 0, ntoreturn)
name = next(iter(cmd))
duration = None
publish = listeners.enabled_for_commands
if publish:
start = datetime.datetime.now()
request_id, msg, max_doc_size = query.get_message(slave_ok,
sock_info.is_mongos)
if publish:
encoding_duration = datetime.datetime.now() - start
listeners.publish_command_start(cmd,
namespace.split('.', 1)[0], request_id,
sock_info.address)
start = datetime.datetime.now()
sock_info.send_message(msg, max_doc_size)
response = sock_info.receive_message(1, request_id)
try:
result = _unpack_response(response, None, codec_options)
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
if isinstance(exc, (NotMasterError, OperationFailure)):
failure = exc.details
else:
failure = _convert_exception(exc)
listeners.publish_command_failure(duration, failure, name,
request_id, sock_info.address)
raise
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
listeners.publish_command_success(duration, result, name, request_id,
sock_info.address)
return result
def run_operation(self, sock_info, operation, set_secondary_okay,
listeners, unpack_res):
"""Run a _Query or _GetMore operation and return a Response object.
This method is used only to run _Query/_GetMore operations from
cursors.
Can raise ConnectionFailure, OperationFailure, etc.
:Parameters:
- `operation`: A _Query or _GetMore object.
- `set_secondary_okay`: Pass to operation.get_message.
- `all_credentials`: dict, maps auth source to MongoCredential.
- `listeners`: Instance of _EventListeners or None.
- `unpack_res`: A callable that decodes the wire protocol response.
"""
duration = None
publish = listeners.enabled_for_commands
if publish:
start = datetime.now()
use_cmd = operation.use_command(sock_info)
more_to_come = (operation.sock_mgr and operation.sock_mgr.more_to_come)
if more_to_come:
request_id = 0
else:
message = operation.get_message(set_secondary_okay, sock_info,
use_cmd)
request_id, data, max_doc_size = self._split_message(message)
if publish:
cmd, dbn = operation.as_command(sock_info)
listeners.publish_command_start(cmd,
dbn,
request_id,
sock_info.address,
service_id=sock_info.service_id)
start = datetime.now()
try:
if more_to_come:
reply = sock_info.receive_message(None)
else:
sock_info.send_message(data, max_doc_size)
reply = sock_info.receive_message(request_id)
# Unpack and check for command errors.
if use_cmd:
user_fields = _CURSOR_DOC_FIELDS
legacy_response = False
else:
user_fields = None
legacy_response = True
docs = unpack_res(reply,
operation.cursor_id,
operation.codec_options,
legacy_response=legacy_response,
user_fields=user_fields)
if use_cmd:
first = docs[0]
operation.client._process_response(first, operation.session)
_check_command_response(first, sock_info.max_wire_version)
except Exception as exc:
if publish:
duration = datetime.now() - start
if isinstance(exc, (NotPrimaryError, OperationFailure)):
failure = exc.details
else:
failure = _convert_exception(exc)
listeners.publish_command_failure(
duration,
failure,
operation.name,
request_id,
sock_info.address,
service_id=sock_info.service_id)
raise
if publish:
duration = datetime.now() - start
# Must publish in find / getMore / explain command response
# format.
if use_cmd:
res = docs[0]
elif operation.name == "explain":
res = docs[0] if docs else {}
else:
res = {
"cursor": {
"id": reply.cursor_id,
"ns": operation.namespace()
},
"ok": 1
}
if operation.name == "find":
res["cursor"]["firstBatch"] = docs
else:
res["cursor"]["nextBatch"] = docs
listeners.publish_command_success(duration,
res,
operation.name,
request_id,
sock_info.address,
service_id=sock_info.service_id)
# Decrypt response.
client = operation.client
if client and client._encrypter:
if use_cmd:
decrypted = client._encrypter.decrypt(
reply.raw_command_response())
docs = _decode_all_selective(decrypted,
operation.codec_options,
user_fields)
if client._should_pin_cursor(operation.session) or operation.exhaust:
sock_info.pin_cursor()
if isinstance(reply, _OpMsg):
# In OP_MSG, the server keeps sending only if the
# more_to_come flag is set.
more_to_come = reply.more_to_come
else:
# In OP_REPLY, the server keeps sending until cursor_id is 0.
more_to_come = bool(operation.exhaust and reply.cursor_id)
if operation.sock_mgr:
operation.sock_mgr.update_exhaust(more_to_come)
response = PinnedResponse(data=reply,
address=self._description.address,
socket_info=sock_info,
duration=duration,
request_id=request_id,
from_command=use_cmd,
docs=docs,
more_to_come=more_to_come)
else:
response = Response(data=reply,
address=self._description.address,
duration=duration,
request_id=request_id,
from_command=use_cmd,
docs=docs)
return response
def __send_message(self, operation):
"""Send a query or getmore operation and handles the response.
If operation is ``None`` this is an exhaust cursor, which reads
the next result batch off the exhaust socket instead of
sending getMore messages to the server.
Can raise ConnectionFailure.
"""
client = self.__collection.database.client
listeners = client._event_listeners
publish = listeners.enabled_for_commands
from_command = False
start = datetime.datetime.now()
def duration(): return datetime.datetime.now() - start
if operation:
kwargs = {
"read_preference": self.__read_preference,
"exhaust": self.__exhaust,
}
if self.__address is not None:
kwargs["address"] = self.__address
try:
response = client._send_message_with_response(operation,
**kwargs)
self.__address = response.address
if self.__exhaust:
# 'response' is an ExhaustResponse.
self.__exhaust_mgr = _SocketManager(response.socket_info,
response.pool)
cmd_name = operation.name
reply = response.data
rqst_id = response.request_id
from_command = response.from_command
except AutoReconnect:
# Don't try to send kill cursors on another socket
# or to another server. It can cause a _pinValue
# assertion on some server releases if we get here
# due to a socket timeout.
self.__killed = True
raise
else:
# Exhaust cursor - no getMore message.
rqst_id = 0
cmd_name = 'getMore'
if publish:
# Fake a getMore command.
cmd = SON([('getMore', self.__id),
('collection', self.__collection.name)])
if self.__batch_size:
cmd['batchSize'] = self.__batch_size
if self.__max_time_ms:
cmd['maxTimeMS'] = self.__max_time_ms
listeners.publish_command_start(
cmd, self.__collection.database.name, 0, self.__address)
try:
reply = self.__exhaust_mgr.sock.receive_message(None)
except Exception as exc:
if publish:
listeners.publish_command_failure(
duration(), _convert_exception(exc), cmd_name, rqst_id,
self.__address)
if isinstance(exc, ConnectionFailure):
self.__die()
raise
try:
docs = self._unpack_response(response=reply,
cursor_id=self.__id,
codec_options=self.__codec_options)
if from_command:
first = docs[0]
client._receive_cluster_time(first, self.__session)
helpers._check_command_response(first)
except OperationFailure as exc:
self.__killed = True
# Make sure exhaust socket is returned immediately, if necessary.
self.__die()
if publish:
listeners.publish_command_failure(
duration(), exc.details, cmd_name, rqst_id, self.__address)
# If this is a tailable cursor the error is likely
# due to capped collection roll over. Setting
# self.__killed to True ensures Cursor.alive will be
# False. No need to re-raise.
if self.__query_flags & _QUERY_OPTIONS["tailable_cursor"]:
return
raise
except NotMasterError as exc:
# Don't send kill cursors to another server after a "not master"
# error. It's completely pointless.
self.__killed = True
# Make sure exhaust socket is returned immediately, if necessary.
self.__die()
if publish:
listeners.publish_command_failure(
duration(), exc.details, cmd_name, rqst_id, self.__address)
client._reset_server_and_request_check(self.__address)
raise
except Exception as exc:
if publish:
listeners.publish_command_failure(
duration(), _convert_exception(exc), cmd_name, rqst_id,
self.__address)
raise
if publish:
# Must publish in find / getMore / explain command response format.
if from_command:
res = docs[0]
elif cmd_name == "explain":
res = docs[0] if reply.number_returned else {}
else:
res = {"cursor": {"id": reply.cursor_id,
"ns": self.__collection.full_name},
"ok": 1}
if cmd_name == "find":
res["cursor"]["firstBatch"] = docs
else:
res["cursor"]["nextBatch"] = docs
listeners.publish_command_success(
duration(), res, cmd_name, rqst_id, self.__address)
if from_command and cmd_name != "explain":
cursor = docs[0]['cursor']
self.__id = cursor['id']
if cmd_name == 'find':
documents = cursor['firstBatch']
else:
documents = cursor['nextBatch']
self.__data = deque(documents)
self.__retrieved += len(documents)
else:
self.__id = reply.cursor_id
self.__data = deque(docs)
self.__retrieved += reply.number_returned
if self.__id == 0:
self.__killed = True
if self.__limit and self.__id and self.__limit <= self.__retrieved:
self.__die()
# Don't wait for garbage collection to call __del__, return the
# socket to the pool now.
if self.__exhaust and self.__id == 0:
self.__exhaust_mgr.close()
def command(sock_info, dbname, spec, secondary_ok, is_mongos,
read_preference, codec_options, session, client, check=True,
allowable_errors=None, address=None,
check_keys=False, listeners=None, max_bson_size=None,
read_concern=None,
parse_write_concern_error=False,
collation=None,
compression_ctx=None,
use_op_msg=False,
unacknowledged=False,
user_fields=None,
exhaust_allowed=False):
"""Execute a command over the socket, or raise socket.error.
:Parameters:
- `sock`: a raw socket instance
- `dbname`: name of the database on which to run the command
- `spec`: a command document as an ordered dict type, eg SON.
- `secondary_ok`: whether to set the secondaryOkay wire protocol bit
- `is_mongos`: are we connected to a mongos?
- `read_preference`: a read preference
- `codec_options`: a CodecOptions instance
- `session`: optional ClientSession instance.
- `client`: optional MongoClient instance for updating $clusterTime.
- `check`: raise OperationFailure if there are errors
- `allowable_errors`: errors to ignore if `check` is True
- `address`: the (host, port) of `sock`
- `check_keys`: if True, check `spec` for invalid keys
- `listeners`: An instance of :class:`~pymongo.monitoring.EventListeners`
- `max_bson_size`: The maximum encoded bson size for this server
- `read_concern`: The read concern for this command.
- `parse_write_concern_error`: Whether to parse the ``writeConcernError``
field in the command response.
- `collation`: The collation for this command.
- `compression_ctx`: optional compression Context.
- `use_op_msg`: True if we should use OP_MSG.
- `unacknowledged`: True if this is an unacknowledged command.
- `user_fields` (optional): Response fields that should be decoded
using the TypeDecoders from codec_options, passed to
bson._decode_all_selective.
- `exhaust_allowed`: True if we should enable OP_MSG exhaustAllowed.
"""
name = next(iter(spec))
ns = dbname + '.$cmd'
flags = 4 if secondary_ok else 0
speculative_hello = False
# Publish the original command document, perhaps with lsid and $clusterTime.
orig = spec
if is_mongos and not use_op_msg:
spec = message._maybe_add_read_preference(spec, read_preference)
if read_concern and not (session and session.in_transaction):
if read_concern.level:
spec['readConcern'] = read_concern.document
if session:
session._update_read_concern(spec, sock_info)
if collation is not None:
spec['collation'] = collation
publish = listeners is not None and listeners.enabled_for_commands
if publish:
start = datetime.datetime.now()
speculative_hello = _is_speculative_authenticate(name, spec)
if compression_ctx and name.lower() in _NO_COMPRESSION:
compression_ctx = None
if (client and client._encrypter and
not client._encrypter._bypass_auto_encryption):
spec = orig = client._encrypter.encrypt(
dbname, spec, check_keys, codec_options)
# We already checked the keys, no need to do it again.
check_keys = False
if use_op_msg:
flags = _OpMsg.MORE_TO_COME if unacknowledged else 0
flags |= _OpMsg.EXHAUST_ALLOWED if exhaust_allowed else 0
request_id, msg, size, max_doc_size = message._op_msg(
flags, spec, dbname, read_preference, secondary_ok, check_keys,
codec_options, ctx=compression_ctx)
# If this is an unacknowledged write then make sure the encoded doc(s)
# are small enough, otherwise rely on the server to return an error.
if (unacknowledged and max_bson_size is not None and
max_doc_size > max_bson_size):
message._raise_document_too_large(name, size, max_bson_size)
else:
request_id, msg, size = message._query(
flags, ns, 0, -1, spec, None, codec_options, check_keys,
compression_ctx)
if (max_bson_size is not None
and size > max_bson_size + message._COMMAND_OVERHEAD):
message._raise_document_too_large(
name, size, max_bson_size + message._COMMAND_OVERHEAD)
if publish:
encoding_duration = datetime.datetime.now() - start
listeners.publish_command_start(orig, dbname, request_id, address,
service_id=sock_info.service_id)
start = datetime.datetime.now()
try:
sock_info.sock.sendall(msg)
if use_op_msg and unacknowledged:
# Unacknowledged, fake a successful command response.
reply = None
response_doc = {"ok": 1}
else:
reply = receive_message(sock_info, request_id)
sock_info.more_to_come = reply.more_to_come
unpacked_docs = reply.unpack_response(
codec_options=codec_options, user_fields=user_fields)
response_doc = unpacked_docs[0]
if client:
client._process_response(response_doc, session)
if check:
helpers._check_command_response(
response_doc, sock_info.max_wire_version, allowable_errors,
parse_write_concern_error=parse_write_concern_error)
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
if isinstance(exc, (NotPrimaryError, OperationFailure)):
failure = exc.details
else:
failure = message._convert_exception(exc)
listeners.publish_command_failure(
duration, failure, name, request_id, address,
service_id=sock_info.service_id)
raise
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
listeners.publish_command_success(
duration, response_doc, name, request_id, address,
service_id=sock_info.service_id,
speculative_hello=speculative_hello)
if client and client._encrypter and reply:
decrypted = client._encrypter.decrypt(reply.raw_command_response())
response_doc = _decode_all_selective(decrypted, codec_options,
user_fields)[0]
return response_doc
def command(sock, dbname, spec, slave_ok, is_mongos,
read_preference, codec_options, session, client, check=True,
allowable_errors=None, address=None,
check_keys=False, listeners=None, max_bson_size=None,
read_concern=None,
parse_write_concern_error=False,
collation=None,
compression_ctx=None,
use_op_msg=False,
unacknowledged=False,
user_fields=None):
"""Execute a command over the socket, or raise socket.error.
:Parameters:
- `sock`: a raw socket instance
- `dbname`: name of the database on which to run the command
- `spec`: a command document as an ordered dict type, eg SON.
- `slave_ok`: whether to set the SlaveOkay wire protocol bit
- `is_mongos`: are we connected to a mongos?
- `read_preference`: a read preference
- `codec_options`: a CodecOptions instance
- `session`: optional ClientSession instance.
- `client`: optional MongoClient instance for updating $clusterTime.
- `check`: raise OperationFailure if there are errors
- `allowable_errors`: errors to ignore if `check` is True
- `address`: the (host, port) of `sock`
- `check_keys`: if True, check `spec` for invalid keys
- `listeners`: An instance of :class:`~pymongo.monitoring.EventListeners`
- `max_bson_size`: The maximum encoded bson size for this server
- `read_concern`: The read concern for this command.
- `parse_write_concern_error`: Whether to parse the ``writeConcernError``
field in the command response.
- `collation`: The collation for this command.
- `compression_ctx`: optional compression Context.
- `use_op_msg`: True if we should use OP_MSG.
- `unacknowledged`: True if this is an unacknowledged command.
- `user_fields` (optional): Response fields that should be decoded
using the TypeDecoders from codec_options, passed to
bson._decode_all_selective.
"""
name = next(iter(spec))
ns = dbname + '.$cmd'
flags = 4 if slave_ok else 0
# Publish the original command document, perhaps with lsid and $clusterTime.
orig = spec
if is_mongos and not use_op_msg:
spec = message._maybe_add_read_preference(spec, read_preference)
if read_concern and not (session and session._in_transaction):
if read_concern.level:
spec['readConcern'] = read_concern.document
if (session and session.options.causal_consistency
and session.operation_time is not None):
spec.setdefault(
'readConcern', {})['afterClusterTime'] = session.operation_time
if collation is not None:
spec['collation'] = collation
publish = listeners is not None and listeners.enabled_for_commands
if publish:
start = datetime.datetime.now()
if compression_ctx and name.lower() in _NO_COMPRESSION:
compression_ctx = None
if use_op_msg:
flags = 2 if unacknowledged else 0
request_id, msg, size, max_doc_size = message._op_msg(
flags, spec, dbname, read_preference, slave_ok, check_keys,
codec_options, ctx=compression_ctx)
# If this is an unacknowledged write then make sure the encoded doc(s)
# are small enough, otherwise rely on the server to return an error.
if (unacknowledged and max_bson_size is not None and
max_doc_size > max_bson_size):
message._raise_document_too_large(name, size, max_bson_size)
else:
request_id, msg, size = message.query(
flags, ns, 0, -1, spec, None, codec_options, check_keys,
compression_ctx)
if (max_bson_size is not None
and size > max_bson_size + message._COMMAND_OVERHEAD):
message._raise_document_too_large(
name, size, max_bson_size + message._COMMAND_OVERHEAD)
if publish:
encoding_duration = datetime.datetime.now() - start
listeners.publish_command_start(orig, dbname, request_id, address)
start = datetime.datetime.now()
try:
sock.sendall(msg)
if use_op_msg and unacknowledged:
# Unacknowledged, fake a successful command response.
response_doc = {"ok": 1}
else:
reply = receive_message(sock, request_id)
unpacked_docs = reply.unpack_response(
codec_options=codec_options, user_fields=user_fields)
response_doc = unpacked_docs[0]
if client:
client._process_response(response_doc, session)
if check:
helpers._check_command_response(
response_doc, None, allowable_errors,
parse_write_concern_error=parse_write_concern_error)
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
if isinstance(exc, (NotMasterError, OperationFailure)):
failure = exc.details
else:
failure = message._convert_exception(exc)
listeners.publish_command_failure(
duration, failure, name, request_id, address)
raise
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
listeners.publish_command_success(
duration, response_doc, name, request_id, address)
return response_doc
def __send_message(self, operation):
"""Send a getmore message and handle the response.
"""
client = self.__collection.database.client
listeners = client._event_listeners
publish = listeners.enabled_for_commands
try:
response = client._send_message_with_response(
operation, address=self.__address)
except AutoReconnect:
# Don't try to send kill cursors on another socket
# or to another server. It can cause a _pinValue
# assertion on some server releases if we get here
# due to a socket timeout.
self.__killed = True
raise
cmd_duration = response.duration
rqst_id = response.request_id
from_command = response.from_command
if publish:
start = datetime.datetime.now()
try:
doc = self._unpack_response(response.data, self.__id,
self.__collection.codec_options)
if from_command:
helpers._check_command_response(doc['data'][0])
except OperationFailure as exc:
self.__killed = True
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
listeners.publish_command_failure(duration, exc.details,
"getMore", rqst_id,
self.__address)
raise
except NotMasterError as exc:
# Don't send kill cursors to another server after a "not master"
# error. It's completely pointless.
self.__killed = True
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
listeners.publish_command_failure(duration, exc.details,
"getMore", rqst_id,
self.__address)
client._reset_server_and_request_check(self.address)
raise
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
listeners.publish_command_failure(duration,
_convert_exception(exc),
"getMore", rqst_id,
self.__address)
raise
if from_command:
cursor = doc['data'][0]['cursor']
documents = cursor['nextBatch']
self.__id = cursor['id']
else:
documents = doc["data"]
self.__id = doc["cursor_id"]
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
# Must publish in getMore command response format.
res = {
"cursor": {
"id": self.__id,
"ns": self.__collection.full_name,
"nextBatch": documents
},
"ok": 1
}
listeners.publish_command_success(duration, res, "getMore",
rqst_id, self.__address)
if self.__id == 0:
self.__killed = True
self.__data = deque(documents)
def __send_message(self, operation):
"""Send a getmore message and handle the response.
"""
client = self.__collection.database.client
listeners = client._event_listeners
publish = listeners.enabled_for_commands
try:
response = client._send_message_with_response(
operation, address=self.__address)
except AutoReconnect:
# Don't try to send kill cursors on another socket
# or to another server. It can cause a _pinValue
# assertion on some server releases if we get here
# due to a socket timeout.
self.__killed = True
raise
cmd_duration = response.duration
rqst_id = response.request_id
if publish:
start = datetime.datetime.now()
try:
doc = helpers._unpack_response(response.data,
self.__id,
self.__collection.codec_options)
except OperationFailure as exc:
self.__killed = True
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
listeners.publish_command_failure(
duration, exc.details, "getMore", rqst_id, self.__address)
raise
except NotMasterError as exc:
# Don't send kill cursors to another server after a "not master"
# error. It's completely pointless.
self.__killed = True
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
listeners.publish_command_failure(
duration, exc.details, "getMore", rqst_id, self.__address)
client._reset_server_and_request_check(self.address)
raise
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
listeners.publish_command_failure(
duration, _convert_exception(exc), "getMore", rqst_id,
self.__address)
raise
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
# Must publish in getMore command response format.
res = {"cursor": {"id": doc["cursor_id"],
"ns": self.__collection.full_name,
"nextBatch": doc["data"]},
"ok": 1}
listeners.publish_command_success(
duration, res, "getMore", rqst_id, self.__address)
self.__id = doc["cursor_id"]
if self.__id == 0:
self.__killed = True
self.__retrieved += doc["number_returned"]
self.__data = deque(doc["data"])
def command(sock, dbname, spec, slave_ok, is_mongos,
read_preference, codec_options, check=True,
allowable_errors=None, address=None,
check_keys=False, listeners=None, max_bson_size=None,
read_concern=DEFAULT_READ_CONCERN):
"""Execute a command over the socket, or raise socket.error.
:Parameters:
- `sock`: a raw socket instance
- `dbname`: name of the database on which to run the command
- `spec`: a command document as a dict, SON, or mapping object
- `slave_ok`: whether to set the SlaveOkay wire protocol bit
- `is_mongos`: are we connected to a mongos?
- `read_preference`: a read preference
- `codec_options`: a CodecOptions instance
- `check`: raise OperationFailure if there are errors
- `allowable_errors`: errors to ignore if `check` is True
- `address`: the (host, port) of `sock`
- `check_keys`: if True, check `spec` for invalid keys
- `listeners`: An instance of :class:`~pymongo.monitoring.EventListeners`
- `max_bson_size`: The maximum encoded bson size for this server
- `read_concern`: The read concern for this command.
"""
name = next(iter(spec))
ns = dbname + '.$cmd'
flags = 4 if slave_ok else 0
# Publish the original command document.
orig = spec
if is_mongos:
spec = message._maybe_add_read_preference(spec, read_preference)
if read_concern.level:
spec['readConcern'] = read_concern.document
publish = listeners is not None and listeners.enabled_for_commands
if publish:
start = datetime.datetime.now()
request_id, msg, size = message.query(flags, ns, 0, -1, spec,
None, codec_options, check_keys)
if (max_bson_size is not None
and size > max_bson_size + message._COMMAND_OVERHEAD):
message._raise_document_too_large(
name, size, max_bson_size + message._COMMAND_OVERHEAD)
if publish:
encoding_duration = datetime.datetime.now() - start
listeners.publish_command_start(orig, dbname, request_id, address)
start = datetime.datetime.now()
try:
sock.sendall(msg)
response = receive_message(sock, 1, request_id)
unpacked = helpers._unpack_response(
response, codec_options=codec_options)
response_doc = unpacked['data'][0]
if check:
helpers._check_command_response(response_doc, None, allowable_errors)
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
if isinstance(exc, (NotMasterError, OperationFailure)):
failure = exc.details
else:
failure = message._convert_exception(exc)
listeners.publish_command_failure(
duration, failure, name, request_id, address)
raise
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
listeners.publish_command_success(
duration, response_doc, name, request_id, address)
return response_doc
def run_operation_with_response(self, sock_info, operation, set_slave_okay,
listeners, exhaust, unpack_res):
"""Run a _Query or _GetMore operation and return a Response object.
This method is used only to run _Query/_GetMore operations from
cursors.
Can raise ConnectionFailure, OperationFailure, etc.
:Parameters:
- `operation`: A _Query or _GetMore object.
- `set_slave_okay`: Pass to operation.get_message.
- `all_credentials`: dict, maps auth source to MongoCredential.
- `listeners`: Instance of _EventListeners or None.
- `exhaust`: If True, then this is an exhaust cursor operation.
- `unpack_res`: A callable that decodes the wire protocol response.
"""
duration = None
publish = listeners.enabled_for_commands
if publish:
start = datetime.now()
send_message = not operation.exhaust_mgr
if send_message:
use_cmd = operation.use_command(sock_info, exhaust)
message = operation.get_message(set_slave_okay, sock_info, use_cmd)
request_id, data, max_doc_size = self._split_message(message)
else:
use_cmd = False
request_id = 0
if publish:
cmd, dbn = operation.as_command(sock_info)
listeners.publish_command_start(cmd, dbn, request_id,
sock_info.address)
start = datetime.now()
try:
if send_message:
sock_info.send_message(data, max_doc_size)
reply = sock_info.receive_message(request_id)
else:
reply = sock_info.receive_message(None)
# Unpack and check for command errors.
if use_cmd:
user_fields = _CURSOR_DOC_FIELDS
legacy_response = False
else:
user_fields = None
legacy_response = True
docs = unpack_res(reply,
operation.cursor_id,
operation.codec_options,
legacy_response=legacy_response,
user_fields=user_fields)
if use_cmd:
first = docs[0]
operation.client._process_response(first, operation.session)
_check_command_response(first)
except Exception as exc:
if publish:
duration = datetime.now() - start
if isinstance(exc, (NotMasterError, OperationFailure)):
failure = exc.details
else:
failure = _convert_exception(exc)
listeners.publish_command_failure(duration, failure,
operation.name, request_id,
sock_info.address)
raise
if publish:
duration = datetime.now() - start
# Must publish in find / getMore / explain command response
# format.
if use_cmd:
res = docs[0]
elif operation.name == "explain":
res = docs[0] if docs else {}
else:
res = {
"cursor": {
"id": reply.cursor_id,
"ns": operation.namespace()
},
"ok": 1
}
if operation.name == "find":
res["cursor"]["firstBatch"] = docs
else:
res["cursor"]["nextBatch"] = docs
listeners.publish_command_success(duration, res, operation.name,
request_id, sock_info.address)
# Decrypt response.
client = operation.client
if client and client._encrypter:
if use_cmd:
decrypted = client._encrypter.decrypt(
reply.raw_command_response())
docs = _decode_all_selective(decrypted,
operation.codec_options,
user_fields)
if exhaust:
response = ExhaustResponse(data=reply,
address=self._description.address,
socket_info=sock_info,
pool=self._pool,
duration=duration,
request_id=request_id,
from_command=use_cmd,
docs=docs)
else:
response = Response(data=reply,
address=self._description.address,
duration=duration,
request_id=request_id,
from_command=use_cmd,
docs=docs)
return response
def send_message_with_response(self,
operation,
set_slave_okay,
all_credentials,
listeners,
exhaust=False):
"""Send a message to MongoDB and return a Response object.
Can raise ConnectionFailure.
:Parameters:
- `operation`: A _Query or _GetMore object.
- `set_slave_okay`: Pass to operation.get_message.
- `all_credentials`: dict, maps auth source to MongoCredential.
- `exhaust` (optional): If True, the socket used stays checked out.
It is returned along with its Pool in the Response.
"""
with self.get_socket(all_credentials, exhaust) as sock_info:
duration = None
publish = listeners.enabled_for_commands
if publish:
start = datetime.now()
use_find_cmd = False
if sock_info.max_wire_version >= 4:
if not exhaust:
use_find_cmd = True
elif (isinstance(operation, _Query)
and not operation.read_concern.ok_for_legacy):
raise ConfigurationError(
'read concern level of %s is not valid '
'with a max wire version of %d.' %
(operation.read_concern.level, sock_info.max_wire_version))
message = operation.get_message(set_slave_okay,
sock_info.is_mongos, use_find_cmd)
request_id, data, max_doc_size = self._split_message(message)
if publish:
encoding_duration = datetime.now() - start
cmd, dbn = operation.as_command()
listeners.publish_command_start(cmd, dbn, request_id,
sock_info.address)
start = datetime.now()
try:
sock_info.send_message(data, max_doc_size)
response_data = sock_info.receive_message(1, request_id)
except Exception as exc:
if publish:
duration = (datetime.now() - start) + encoding_duration
failure = _convert_exception(exc)
listeners.publish_command_failure(duration, failure,
next(iter(cmd)),
request_id,
sock_info.address)
raise
if publish:
duration = (datetime.now() - start) + encoding_duration
if exhaust:
return ExhaustResponse(data=response_data,
address=self._description.address,
socket_info=sock_info,
pool=self._pool,
duration=duration,
request_id=request_id,
from_command=use_find_cmd)
else:
return Response(data=response_data,
address=self._description.address,
duration=duration,
request_id=request_id,
from_command=use_find_cmd)
def _process_kill_cursors_queue(self):
"""Process any pending kill cursors requests."""
address_to_cursor_ids = defaultdict(list)
# Other threads or the GC may append to the queue concurrently.
while True:
try:
address, cursor_ids = self.__kill_cursors_queue.pop()
except IndexError:
break
address_to_cursor_ids[address].extend(cursor_ids)
# Don't re-open topology if it's closed and there's no pending cursors.
if address_to_cursor_ids:
listeners = self._event_listeners
publish = listeners.enabled_for_commands
topology = self._get_topology()
for address, cursor_ids in address_to_cursor_ids.items():
try:
if address:
# address could be a tuple or _CursorAddress, but
# select_server_by_address needs (host, port).
server = topology.select_server_by_address(
tuple(address))
else:
# Application called close_cursor() with no address.
server = topology.select_server(
writable_server_selector)
if publish:
start = datetime.datetime.now()
data = message.kill_cursors(cursor_ids)
if publish:
duration = datetime.datetime.now() - start
try:
dbname, collname = address.namespace.split(".", 1)
except AttributeError:
dbname = collname = 'OP_KILL_CURSORS'
command = SON([('killCursors', collname),
('cursors', cursor_ids)])
listeners.publish_command_start(
command, dbname, data[0], address)
start = datetime.datetime.now()
try:
server.send_message(data, self.__all_credentials)
except Exception as exc:
if publish:
dur = (datetime.datetime.now() - start) + duration
listeners.publish_command_failure(
dur, message._convert_exception(exc),
'killCursors', data[0], address)
raise
if publish:
duration = (datetime.datetime.now() - start) + duration
# OP_KILL_CURSORS returns no reply, fake one.
reply = {'cursorsUnknown': cursor_ids, 'ok': 1}
listeners.publish_command_success(
duration, reply, 'killCursors', data[0], address)
except ConnectionFailure as exc:
warnings.warn("couldn't close cursor on %s: %s"
% (address, exc))
def run_operation_with_response(
self,
sock_info,
operation,
set_slave_okay,
listeners,
exhaust,
unpack_res):
"""Run a _Query or _GetMore operation and return a Response object.
This method is used only to run _Query/_GetMore operations from
cursors.
Can raise ConnectionFailure, OperationFailure, etc.
:Parameters:
- `operation`: A _Query or _GetMore object.
- `set_slave_okay`: Pass to operation.get_message.
- `all_credentials`: dict, maps auth source to MongoCredential.
- `listeners`: Instance of _EventListeners or None.
- `exhaust`: If True, then this is an exhaust cursor operation.
- `unpack_res`: A callable that decodes the wire protocol response.
"""
duration = None
publish = listeners.enabled_for_commands
if publish:
start = datetime.now()
send_message = not operation.exhaust_mgr
if send_message:
use_cmd = operation.use_command(sock_info, exhaust)
message = operation.get_message(
set_slave_okay, sock_info, use_cmd)
request_id, data, max_doc_size = self._split_message(message)
else:
use_cmd = False
request_id = 0
if publish:
cmd, dbn = operation.as_command(sock_info)
listeners.publish_command_start(
cmd, dbn, request_id, sock_info.address)
start = datetime.now()
try:
if send_message:
sock_info.send_message(data, max_doc_size)
reply = sock_info.receive_message(request_id)
else:
reply = sock_info.receive_message(None)
# Unpack and check for command errors.
if use_cmd:
user_fields = _CURSOR_DOC_FIELDS
legacy_response = False
else:
user_fields = None
legacy_response = True
docs = unpack_res(reply, operation.cursor_id,
operation.codec_options,
legacy_response=legacy_response,
user_fields=user_fields)
if use_cmd:
first = docs[0]
operation.client._process_response(
first, operation.session)
_check_command_response(first)
except Exception as exc:
if publish:
duration = datetime.now() - start
if isinstance(exc, (NotMasterError, OperationFailure)):
failure = exc.details
else:
failure = _convert_exception(exc)
listeners.publish_command_failure(
duration, failure, operation.name,
request_id, sock_info.address)
raise
if publish:
duration = datetime.now() - start
# Must publish in find / getMore / explain command response
# format.
if use_cmd:
res = docs[0]
elif operation.name == "explain":
res = docs[0] if docs else {}
else:
res = {"cursor": {"id": reply.cursor_id,
"ns": operation.namespace()},
"ok": 1}
if operation.name == "find":
res["cursor"]["firstBatch"] = docs
else:
res["cursor"]["nextBatch"] = docs
listeners.publish_command_success(
duration, res, operation.name, request_id,
sock_info.address)
if exhaust:
response = ExhaustResponse(
data=reply,
address=self._description.address,
socket_info=sock_info,
pool=self._pool,
duration=duration,
request_id=request_id,
from_command=use_cmd,
docs=docs)
else:
response = Response(
data=reply,
address=self._description.address,
duration=duration,
request_id=request_id,
from_command=use_cmd,
docs=docs)
return response
def send_message_with_response(
self,
operation,
set_slave_okay,
all_credentials,
listeners,
exhaust=False):
"""Send a message to MongoDB and return a Response object.
Can raise ConnectionFailure.
:Parameters:
- `operation`: A _Query or _GetMore object.
- `set_slave_okay`: Pass to operation.get_message.
- `all_credentials`: dict, maps auth source to MongoCredential.
- `listeners`: Instance of _EventListeners or None.
- `exhaust` (optional): If True, the socket used stays checked out.
It is returned along with its Pool in the Response.
"""
with self.get_socket(all_credentials, exhaust) as sock_info:
duration = None
publish = listeners.enabled_for_commands
if publish:
start = datetime.now()
use_find_cmd = operation.use_command(sock_info, exhaust)
message = operation.get_message(
set_slave_okay, sock_info, use_find_cmd)
request_id, data, max_doc_size = self._split_message(message)
if publish:
encoding_duration = datetime.now() - start
cmd, dbn = operation.as_command(sock_info)
listeners.publish_command_start(
cmd, dbn, request_id, sock_info.address)
start = datetime.now()
try:
sock_info.send_message(data, max_doc_size)
reply = sock_info.receive_message(request_id)
except Exception as exc:
if publish:
duration = (datetime.now() - start) + encoding_duration
failure = _convert_exception(exc)
listeners.publish_command_failure(
duration, failure, next(iter(cmd)), request_id,
sock_info.address)
raise
if publish:
duration = (datetime.now() - start) + encoding_duration
if exhaust:
return ExhaustResponse(
data=reply,
address=self._description.address,
socket_info=sock_info,
pool=self._pool,
duration=duration,
request_id=request_id,
from_command=use_find_cmd)
else:
return Response(
data=reply,
address=self._description.address,
duration=duration,
request_id=request_id,
from_command=use_find_cmd)
def __send_message(self, operation):
"""Send a getmore message and handle the response.
"""
def kill():
self.__killed = True
self.__end_session(True)
client = self.__collection.database.client
listeners = client._event_listeners
publish = listeners.enabled_for_commands
start = datetime.datetime.now()
def duration(): return datetime.datetime.now() - start
try:
response = client._send_message_with_response(
operation, address=self.__address)
except AutoReconnect:
# Don't try to send kill cursors on another socket
# or to another server. It can cause a _pinValue
# assertion on some server releases if we get here
# due to a socket timeout.
kill()
raise
rqst_id = response.request_id
from_command = response.from_command
reply = response.data
try:
docs = self._unpack_response(reply,
self.__id,
self.__collection.codec_options)
if from_command:
first = docs[0]
client._receive_cluster_time(first, self.__session)
helpers._check_command_response(first)
except OperationFailure as exc:
kill()
if publish:
listeners.publish_command_failure(
duration(), exc.details, "getMore", rqst_id, self.__address)
raise
except NotMasterError as exc:
# Don't send kill cursors to another server after a "not master"
# error. It's completely pointless.
kill()
if publish:
listeners.publish_command_failure(
duration(), exc.details, "getMore", rqst_id, self.__address)
client._reset_server_and_request_check(self.address)
raise
except Exception as exc:
if publish:
listeners.publish_command_failure(
duration(), _convert_exception(exc), "getMore", rqst_id,
self.__address)
raise
if from_command:
cursor = docs[0]['cursor']
documents = cursor['nextBatch']
self.__id = cursor['id']
if publish:
listeners.publish_command_success(
duration(), docs[0], "getMore", rqst_id,
self.__address)
else:
documents = docs
self.__id = reply.cursor_id
if publish:
# Must publish in getMore command response format.
res = {"cursor": {"id": self.__id,
"ns": self.__collection.full_name,
"nextBatch": documents},
"ok": 1}
listeners.publish_command_success(
duration(), res, "getMore", rqst_id, self.__address)
if self.__id == 0:
kill()
self.__data = deque(documents)
def command(sock,
dbname,
spec,
slave_ok,
is_mongos,
read_preference,
codec_options,
session,
client,
check=True,
allowable_errors=None,
address=None,
check_keys=False,
listeners=None,
max_bson_size=None,
read_concern=None,
parse_write_concern_error=False,
collation=None,
compression_ctx=None,
use_op_msg=False,
unacknowledged=False):
"""Execute a command over the socket, or raise socket.error.
:Parameters:
- `sock`: a raw socket instance
- `dbname`: name of the database on which to run the command
- `spec`: a command document as an ordered dict type, eg SON.
- `slave_ok`: whether to set the SlaveOkay wire protocol bit
- `is_mongos`: are we connected to a mongos?
- `read_preference`: a read preference
- `codec_options`: a CodecOptions instance
- `session`: optional ClientSession instance.
- `client`: optional MongoClient instance for updating $clusterTime.
- `check`: raise OperationFailure if there are errors
- `allowable_errors`: errors to ignore if `check` is True
- `address`: the (host, port) of `sock`
- `check_keys`: if True, check `spec` for invalid keys
- `listeners`: An instance of :class:`~pymongo.monitoring.EventListeners`
- `max_bson_size`: The maximum encoded bson size for this server
- `read_concern`: The read concern for this command.
- `parse_write_concern_error`: Whether to parse the ``writeConcernError``
field in the command response.
- `collation`: The collation for this command.
"""
name = next(iter(spec))
ns = dbname + '.$cmd'
flags = 4 if slave_ok else 0
# Publish the original command document, perhaps with lsid and $clusterTime.
orig = spec
if is_mongos and not use_op_msg:
spec = message._maybe_add_read_preference(spec, read_preference)
if read_concern and not (session and session._in_transaction):
if read_concern.level:
spec['readConcern'] = read_concern.document
if (session and session.options.causal_consistency
and session.operation_time is not None):
spec.setdefault('readConcern',
{})['afterClusterTime'] = session.operation_time
if collation is not None:
spec['collation'] = collation
publish = listeners is not None and listeners.enabled_for_commands
if publish:
start = datetime.datetime.now()
if compression_ctx and name.lower() in _NO_COMPRESSION:
compression_ctx = None
if use_op_msg:
flags = 2 if unacknowledged else 0
request_id, msg, size, max_doc_size = message._op_msg(
flags,
spec,
dbname,
read_preference,
slave_ok,
check_keys,
codec_options,
ctx=compression_ctx)
# If this is an unacknowledged write then make sure the encoded doc(s)
# are small enough, otherwise rely on the server to return an error.
if (unacknowledged and max_bson_size is not None
and max_doc_size > max_bson_size):
message._raise_document_too_large(name, size, max_bson_size)
else:
request_id, msg, size = message.query(flags, ns, 0, -1, spec, None,
codec_options, check_keys,
compression_ctx)
if (max_bson_size is not None
and size > max_bson_size + message._COMMAND_OVERHEAD):
message._raise_document_too_large(
name, size, max_bson_size + message._COMMAND_OVERHEAD)
if publish:
encoding_duration = datetime.datetime.now() - start
listeners.publish_command_start(orig, dbname, request_id, address)
start = datetime.datetime.now()
try:
sock.sendall(msg)
if use_op_msg and unacknowledged:
# Unacknowledged, fake a successful command response.
response_doc = {"ok": 1}
else:
reply = receive_message(sock, request_id)
unpacked_docs = reply.unpack_response(codec_options=codec_options)
response_doc = unpacked_docs[0]
if client:
client._receive_cluster_time(response_doc, session)
if check:
helpers._check_command_response(
response_doc,
None,
allowable_errors,
parse_write_concern_error=parse_write_concern_error)
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
if isinstance(exc, (NotMasterError, OperationFailure)):
failure = exc.details
else:
failure = message._convert_exception(exc)
listeners.publish_command_failure(duration, failure, name,
request_id, address)
raise
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
listeners.publish_command_success(duration, response_doc, name,
request_id, address)
return response_doc
def _process_kill_cursors_queue(self):
"""Process any pending kill cursors requests."""
address_to_cursor_ids = defaultdict(list)
# Other threads or the GC may append to the queue concurrently.
while True:
try:
address, cursor_ids = self.__kill_cursors_queue.pop()
except IndexError:
break
address_to_cursor_ids[address].extend(cursor_ids)
# Don't re-open topology if it's closed and there's no pending cursors.
if address_to_cursor_ids:
listeners = self._event_listeners
publish = listeners.enabled_for_commands
topology = self._get_topology()
for address, cursor_ids in address_to_cursor_ids.items():
try:
if address:
# address could be a tuple or _CursorAddress, but
# select_server_by_address needs (host, port).
server = topology.select_server_by_address(
tuple(address))
else:
# Application called close_cursor() with no address.
server = topology.select_server(
writable_server_selector)
if publish:
start = datetime.datetime.now()
data = message.kill_cursors(cursor_ids)
if publish:
duration = datetime.datetime.now() - start
try:
dbname, collname = address.namespace.split(".", 1)
except AttributeError:
dbname = collname = 'OP_KILL_CURSORS'
command = SON([('killCursors', collname),
('cursors', cursor_ids)])
listeners.publish_command_start(
command, dbname, data[0], address)
start = datetime.datetime.now()
try:
server.send_message(data, self.__all_credentials)
except Exception as exc:
if publish:
dur = (datetime.datetime.now() - start) + duration
listeners.publish_command_failure(
dur, message._convert_exception(exc),
'killCursors', data[0], address)
raise
if publish:
duration = (datetime.datetime.now() - start) + duration
# OP_KILL_CURSORS returns no reply, fake one.
reply = {'cursorsUnknown': cursor_ids, 'ok': 1}
listeners.publish_command_success(
duration, reply, 'killCursors', data[0], address)
except ConnectionFailure as exc:
warnings.warn("couldn't close cursor on %s: %s" %
(address, exc))
def send_message_with_response(
self,
operation,
set_slave_okay,
all_credentials,
listeners,
exhaust=False):
"""Send a message to MongoDB and return a Response object.
Can raise ConnectionFailure.
:Parameters:
- `operation`: A _Query or _GetMore object.
- `set_slave_okay`: Pass to operation.get_message.
- `all_credentials`: dict, maps auth source to MongoCredential.
- `exhaust` (optional): If True, the socket used stays checked out.
It is returned along with its Pool in the Response.
"""
with self.get_socket(all_credentials, exhaust) as sock_info:
duration = None
publish = listeners.enabled_for_commands
if publish:
start = datetime.now()
use_find_cmd = False
if sock_info.max_wire_version >= 4:
if not exhaust:
use_find_cmd = True
elif (isinstance(operation, _Query) and
not operation.read_concern.ok_for_legacy):
raise ConfigurationError(
'read concern level of %s is not valid '
'with a max wire version of %d.'
% (operation.read_concern.level,
sock_info.max_wire_version))
message = operation.get_message(
set_slave_okay, sock_info.is_mongos, use_find_cmd)
request_id, data, max_doc_size = self._split_message(message)
if publish:
encoding_duration = datetime.now() - start
cmd, dbn = operation.as_command()
listeners.publish_command_start(
cmd, dbn, request_id, sock_info.address)
start = datetime.now()
try:
sock_info.send_message(data, max_doc_size)
response_data = sock_info.receive_message(1, request_id)
except Exception as exc:
if publish:
duration = (datetime.now() - start) + encoding_duration
failure = _convert_exception(exc)
listeners.publish_command_failure(
duration, failure, next(iter(cmd)), request_id,
sock_info.address)
raise
if publish:
duration = (datetime.now() - start) + encoding_duration
if exhaust:
return ExhaustResponse(
data=response_data,
address=self._description.address,
socket_info=sock_info,
pool=self._pool,
duration=duration,
request_id=request_id,
from_command=use_find_cmd)
else:
return Response(
data=response_data,
address=self._description.address,
duration=duration,
request_id=request_id,
from_command=use_find_cmd)
def __send_message(self, operation):
"""Send a getmore message and handle the response.
"""
def kill():
self.__killed = True
self.__end_session(True)
client = self.__collection.database.client
listeners = client._event_listeners
publish = listeners.enabled_for_commands
start = datetime.datetime.now()
def duration():
return datetime.datetime.now() - start
try:
response = client._send_message_with_response(
operation, address=self.__address)
except AutoReconnect:
# Don't try to send kill cursors on another socket
# or to another server. It can cause a _pinValue
# assertion on some server releases if we get here
# due to a socket timeout.
kill()
raise
rqst_id = response.request_id
from_command = response.from_command
reply = response.data
try:
docs = self._unpack_response(reply, self.__id,
self.__collection.codec_options)
if from_command:
first = docs[0]
client._receive_cluster_time(first, self.__session)
helpers._check_command_response(first)
except OperationFailure as exc:
kill()
if publish:
listeners.publish_command_failure(duration(), exc.details,
"getMore", rqst_id,
self.__address)
raise
except NotMasterError as exc:
# Don't send kill cursors to another server after a "not master"
# error. It's completely pointless.
kill()
if publish:
listeners.publish_command_failure(duration(), exc.details,
"getMore", rqst_id,
self.__address)
client._reset_server_and_request_check(self.address)
raise
except Exception as exc:
if publish:
listeners.publish_command_failure(duration(),
_convert_exception(exc),
"getMore", rqst_id,
self.__address)
raise
if from_command:
cursor = docs[0]['cursor']
documents = cursor['nextBatch']
self.__id = cursor['id']
if publish:
listeners.publish_command_success(duration(), docs[0],
"getMore", rqst_id,
self.__address)
else:
documents = docs
self.__id = reply.cursor_id
if publish:
# Must publish in getMore command response format.
res = {
"cursor": {
"id": self.__id,
"ns": self.__collection.full_name,
"nextBatch": documents
},
"ok": 1
}
listeners.publish_command_success(duration(), res, "getMore",
rqst_id, self.__address)
if self.__id == 0:
kill()
self.__data = deque(documents)
def command(sock,
dbname,
spec,
slave_ok,
is_mongos,
read_preference,
codec_options,
check=True,
allowable_errors=None,
address=None,
check_keys=False,
listeners=None,
max_bson_size=None,
read_concern=DEFAULT_READ_CONCERN):
"""Execute a command over the socket, or raise socket.error.
:Parameters:
- `sock`: a raw socket instance
- `dbname`: name of the database on which to run the command
- `spec`: a command document as a dict, SON, or mapping object
- `slave_ok`: whether to set the SlaveOkay wire protocol bit
- `is_mongos`: are we connected to a mongos?
- `read_preference`: a read preference
- `codec_options`: a CodecOptions instance
- `check`: raise OperationFailure if there are errors
- `allowable_errors`: errors to ignore if `check` is True
- `address`: the (host, port) of `sock`
- `check_keys`: if True, check `spec` for invalid keys
- `listeners`: An instance of :class:`~pymongo.monitoring.EventListeners`
- `max_bson_size`: The maximum encoded bson size for this server
- `read_concern`: The read concern for this command.
"""
name = next(iter(spec))
ns = dbname + '.$cmd'
flags = 4 if slave_ok else 0
# Publish the original command document.
orig = spec
if is_mongos:
spec = message._maybe_add_read_preference(spec, read_preference)
if read_concern.level:
spec['readConcern'] = read_concern.document
publish = listeners is not None and listeners.enabled_for_commands
if publish:
start = datetime.datetime.now()
request_id, msg, size = message.query(flags, ns, 0, -1, spec, None,
codec_options, check_keys)
if (max_bson_size is not None
and size > max_bson_size + message._COMMAND_OVERHEAD):
message._raise_document_too_large(
name, size, max_bson_size + message._COMMAND_OVERHEAD)
if publish:
encoding_duration = datetime.datetime.now() - start
listeners.publish_command_start(orig, dbname, request_id, address)
start = datetime.datetime.now()
try:
sock.sendall(msg)
response = receive_message(sock, 1, request_id)
unpacked = helpers._unpack_response(response,
codec_options=codec_options)
response_doc = unpacked['data'][0]
if check:
helpers._check_command_response(response_doc, None,
allowable_errors)
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
if isinstance(exc, (NotMasterError, OperationFailure)):
failure = exc.details
else:
failure = message._convert_exception(exc)
listeners.publish_command_failure(duration, failure, name,
request_id, address)
raise
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
listeners.publish_command_success(duration, response_doc, name,
request_id, address)
return response_doc
def send_message_with_response(self,
operation,
set_slave_okay,
all_credentials,
listeners,
exhaust=False):
"""Send a message to MongoDB and return a Response object.
Can raise ConnectionFailure.
:Parameters:
- `operation`: A _Query or _GetMore object.
- `set_slave_okay`: Pass to operation.get_message.
- `all_credentials`: dict, maps auth source to MongoCredential.
- `listeners`: Instance of _EventListeners or None.
- `exhaust` (optional): If True, the socket used stays checked out.
It is returned along with its Pool in the Response.
"""
with self.get_socket(all_credentials, exhaust) as sock_info:
duration = None
publish = listeners.enabled_for_commands
if publish:
start = datetime.now()
use_find_cmd = operation.use_command(sock_info, exhaust)
message = operation.get_message(set_slave_okay, sock_info,
use_find_cmd)
request_id, data, max_doc_size = self._split_message(message)
if publish:
encoding_duration = datetime.now() - start
cmd, dbn = operation.as_command(sock_info)
listeners.publish_command_start(cmd, dbn, request_id,
sock_info.address)
start = datetime.now()
try:
sock_info.send_message(data, max_doc_size)
reply = sock_info.receive_message(request_id)
except Exception as exc:
if publish:
duration = (datetime.now() - start) + encoding_duration
failure = _convert_exception(exc)
listeners.publish_command_failure(duration, failure,
next(iter(cmd)),
request_id,
sock_info.address)
raise
if publish:
duration = (datetime.now() - start) + encoding_duration
if exhaust:
return ExhaustResponse(data=reply,
address=self._description.address,
socket_info=sock_info,
pool=self._pool,
duration=duration,
request_id=request_id,
from_command=use_find_cmd)
else:
return Response(data=reply,
address=self._description.address,
duration=duration,
request_id=request_id,
from_command=use_find_cmd)
def command(sock,
dbname,
spec,
slave_ok,
is_mongos,
read_preference,
codec_options,
session,
client,
check=True,
allowable_errors=None,
address=None,
check_keys=False,
listeners=None,
max_bson_size=None,
read_concern=DEFAULT_READ_CONCERN,
parse_write_concern_error=False,
collation=None,
retryable_write=False):
"""Execute a command over the socket, or raise socket.error.
:Parameters:
- `sock`: a raw socket instance
- `dbname`: name of the database on which to run the command
- `spec`: a command document as a dict, SON, or mapping object
- `slave_ok`: whether to set the SlaveOkay wire protocol bit
- `is_mongos`: are we connected to a mongos?
- `read_preference`: a read preference
- `codec_options`: a CodecOptions instance
- `session`: optional ClientSession instance.
- `client`: optional MongoClient instance for updating $clusterTime.
- `check`: raise OperationFailure if there are errors
- `allowable_errors`: errors to ignore if `check` is True
- `address`: the (host, port) of `sock`
- `check_keys`: if True, check `spec` for invalid keys
- `listeners`: An instance of :class:`~pymongo.monitoring.EventListeners`
- `max_bson_size`: The maximum encoded bson size for this server
- `read_concern`: The read concern for this command.
- `parse_write_concern_error`: Whether to parse the ``writeConcernError``
field in the command response.
- `collation`: The collation for this command.
- `retryable_write`: True if this command is a retryable write.
"""
name = next(iter(spec))
ns = dbname + '.$cmd'
flags = 4 if slave_ok else 0
if (client or session) and not isinstance(spec, ORDERED_TYPES):
# Ensure command name remains in first place.
spec = SON(spec)
if session:
spec['lsid'] = session._use_lsid()
if retryable_write:
spec['txnNumber'] = session._transaction_id()
if client:
client._send_cluster_time(spec, session)
# Publish the original command document, perhaps with lsid and $clusterTime.
orig = spec
if is_mongos:
spec = message._maybe_add_read_preference(spec, read_preference)
if read_concern.level:
spec['readConcern'] = read_concern.document
if (session and session.options.causal_consistency
and session.operation_time is not None):
spec.setdefault('readConcern',
{})['afterClusterTime'] = session.operation_time
if collation is not None:
spec['collation'] = collation
publish = listeners is not None and listeners.enabled_for_commands
if publish:
start = datetime.datetime.now()
request_id, msg, size = message.query(flags, ns, 0, -1, spec, None,
codec_options, check_keys)
if (max_bson_size is not None
and size > max_bson_size + message._COMMAND_OVERHEAD):
message._raise_document_too_large(
name, size, max_bson_size + message._COMMAND_OVERHEAD)
if publish:
encoding_duration = datetime.datetime.now() - start
listeners.publish_command_start(orig, dbname, request_id, address)
start = datetime.datetime.now()
try:
sock.sendall(msg)
reply = receive_message(sock, request_id)
unpacked_docs = reply.unpack_response(codec_options=codec_options)
response_doc = unpacked_docs[0]
if client:
client._receive_cluster_time(response_doc)
if session:
session._advance_cluster_time(response_doc.get('$clusterTime'))
session._advance_operation_time(
response_doc.get('operationTime'))
if check:
helpers._check_command_response(
response_doc,
None,
allowable_errors,
parse_write_concern_error=parse_write_concern_error)
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
if isinstance(exc, (NotMasterError, OperationFailure)):
failure = exc.details
else:
failure = message._convert_exception(exc)
listeners.publish_command_failure(duration, failure, name,
request_id, address)
raise
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
listeners.publish_command_success(duration, response_doc, name,
request_id, address)
return response_doc
def __send_message(self, operation):
"""Send a query or getmore operation and handles the response.
If operation is ``None`` this is an exhaust cursor, which reads
the next result batch off the exhaust socket instead of
sending getMore messages to the server.
Can raise ConnectionFailure.
"""
client = self.__collection.database.client
listeners = client._event_listeners
publish = listeners.enabled_for_commands
from_command = False
if operation:
kwargs = {
"read_preference": self.__read_preference,
"exhaust": self.__exhaust,
}
if self.__address is not None:
kwargs["address"] = self.__address
try:
response = client._send_message_with_response(
operation, **kwargs)
self.__address = response.address
if self.__exhaust:
# 'response' is an ExhaustResponse.
self.__exhaust_mgr = _SocketManager(
response.socket_info, response.pool)
cmd_name = operation.name
data = response.data
cmd_duration = response.duration
rqst_id = response.request_id
from_command = response.from_command
except AutoReconnect:
# Don't try to send kill cursors on another socket
# or to another server. It can cause a _pinValue
# assertion on some server releases if we get here
# due to a socket timeout.
self.__killed = True
raise
else:
# Exhaust cursor - no getMore message.
rqst_id = 0
cmd_name = 'getMore'
if publish:
# Fake a getMore command.
cmd = SON([('getMore', self.__id),
('collection', self.__collection.name)])
if self.__batch_size:
cmd['batchSize'] = self.__batch_size
if self.__max_time_ms:
cmd['maxTimeMS'] = self.__max_time_ms
listeners.publish_command_start(
cmd, self.__collection.database.name, 0, self.__address)
start = datetime.datetime.now()
try:
data = self.__exhaust_mgr.sock.receive_message(1, None)
except Exception as exc:
if publish:
duration = datetime.datetime.now() - start
listeners.publish_command_failure(duration,
_convert_exception(exc),
cmd_name, rqst_id,
self.__address)
if isinstance(exc, ConnectionFailure):
self.__die()
raise
if publish:
cmd_duration = datetime.datetime.now() - start
if publish:
start = datetime.datetime.now()
try:
doc = helpers._unpack_response(response=data,
cursor_id=self.__id,
codec_options=self.__codec_options)
if from_command:
helpers._check_command_response(doc['data'][0])
except OperationFailure as exc:
self.__killed = True
# Make sure exhaust socket is returned immediately, if necessary.
self.__die()
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
listeners.publish_command_failure(duration, exc.details,
cmd_name, rqst_id,
self.__address)
# If this is a tailable cursor the error is likely
# due to capped collection roll over. Setting
# self.__killed to True ensures Cursor.alive will be
# False. No need to re-raise.
if self.__query_flags & _QUERY_OPTIONS["tailable_cursor"]:
return
raise
except NotMasterError as exc:
# Don't send kill cursors to another server after a "not master"
# error. It's completely pointless.
self.__killed = True
# Make sure exhaust socket is returned immediately, if necessary.
self.__die()
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
listeners.publish_command_failure(duration, exc.details,
cmd_name, rqst_id,
self.__address)
client._reset_server_and_request_check(self.__address)
raise
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
listeners.publish_command_failure(duration,
_convert_exception(exc),
cmd_name, rqst_id,
self.__address)
raise
if publish:
duration = (datetime.datetime.now() - start) + cmd_duration
# Must publish in find / getMore / explain command response format.
if from_command:
res = doc['data'][0]
elif cmd_name == "explain":
res = doc["data"][0] if doc["number_returned"] else {}
else:
res = {
"cursor": {
"id": doc["cursor_id"],
"ns": self.__collection.full_name
},
"ok": 1
}
if cmd_name == "find":
res["cursor"]["firstBatch"] = doc["data"]
else:
res["cursor"]["nextBatch"] = doc["data"]
listeners.publish_command_success(duration, res, cmd_name, rqst_id,
self.__address)
if from_command and cmd_name != "explain":
cursor = doc['data'][0]['cursor']
self.__id = cursor['id']
if cmd_name == 'find':
documents = cursor['firstBatch']
else:
documents = cursor['nextBatch']
self.__data = deque(documents)
self.__retrieved += len(documents)
else:
self.__id = doc["cursor_id"]
self.__data = deque(doc["data"])
self.__retrieved += doc["number_returned"]
if self.__id == 0:
self.__killed = True
if self.__limit and self.__id and self.__limit <= self.__retrieved:
self.__die()
# Don't wait for garbage collection to call __del__, return the
# socket to the pool now.
if self.__exhaust and self.__id == 0:
self.__exhaust_mgr.close()
def command(sock, dbname, spec, slave_ok, is_mongos,
read_preference, codec_options, session, client, check=True,
allowable_errors=None, address=None,
check_keys=False, listeners=None, max_bson_size=None,
read_concern=None,
parse_write_concern_error=False,
collation=None,
compression_ctx=None):
"""Execute a command over the socket, or raise socket.error.
:Parameters:
- `sock`: a raw socket instance
- `dbname`: name of the database on which to run the command
- `spec`: a command document as a dict, SON, or mapping object
- `slave_ok`: whether to set the SlaveOkay wire protocol bit
- `is_mongos`: are we connected to a mongos?
- `read_preference`: a read preference
- `codec_options`: a CodecOptions instance
- `session`: optional ClientSession instance.
- `client`: optional MongoClient instance for updating $clusterTime.
- `check`: raise OperationFailure if there are errors
- `allowable_errors`: errors to ignore if `check` is True
- `address`: the (host, port) of `sock`
- `check_keys`: if True, check `spec` for invalid keys
- `listeners`: An instance of :class:`~pymongo.monitoring.EventListeners`
- `max_bson_size`: The maximum encoded bson size for this server
- `read_concern`: The read concern for this command.
- `parse_write_concern_error`: Whether to parse the ``writeConcernError``
field in the command response.
- `collation`: The collation for this command.
"""
name = next(iter(spec))
ns = dbname + '.$cmd'
flags = 4 if slave_ok else 0
# Publish the original command document, perhaps with lsid and $clusterTime.
orig = spec
if is_mongos:
spec = message._maybe_add_read_preference(spec, read_preference)
if read_concern:
if read_concern.level:
spec['readConcern'] = read_concern.document
if (session and session.options.causal_consistency
and session.operation_time is not None
and not session._in_transaction):
spec.setdefault(
'readConcern', {})['afterClusterTime'] = session.operation_time
if collation is not None:
spec['collation'] = collation
publish = listeners is not None and listeners.enabled_for_commands
if publish:
start = datetime.datetime.now()
if name.lower() not in _NO_COMPRESSION and compression_ctx:
request_id, msg, size = message.query(
flags, ns, 0, -1, spec, None, codec_options, check_keys, compression_ctx)
else:
request_id, msg, size = message.query(
flags, ns, 0, -1, spec, None, codec_options, check_keys)
if (max_bson_size is not None
and size > max_bson_size + message._COMMAND_OVERHEAD):
message._raise_document_too_large(
name, size, max_bson_size + message._COMMAND_OVERHEAD)
if publish:
encoding_duration = datetime.datetime.now() - start
listeners.publish_command_start(orig, dbname, request_id, address)
start = datetime.datetime.now()
try:
sock.sendall(msg)
reply = receive_message(sock, request_id)
unpacked_docs = reply.unpack_response(codec_options=codec_options)
response_doc = unpacked_docs[0]
if client:
client._receive_cluster_time(response_doc, session)
if check:
helpers._check_command_response(
response_doc, None, allowable_errors,
parse_write_concern_error=parse_write_concern_error)
except Exception as exc:
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
if isinstance(exc, (NotMasterError, OperationFailure)):
failure = exc.details
else:
failure = message._convert_exception(exc)
listeners.publish_command_failure(
duration, failure, name, request_id, address)
raise
if publish:
duration = (datetime.datetime.now() - start) + encoding_duration
listeners.publish_command_success(
duration, response_doc, name, request_id, address)
return response_doc
