def _command(self,
sock_info,
command,
slave_ok=False,
value=1,
check=True,
allowable_errors=None,
read_preference=ReadPreference.PRIMARY,
codec_options=DEFAULT_CODEC_OPTIONS,
write_concern=None,
parse_write_concern_error=False,
**kwargs):
"""Internal command helper."""
if isinstance(command, string_type):
command = SON([(command, value)])
if sock_info.max_wire_version >= 5 and write_concern:
command['writeConcern'] = write_concern.document
command.update(kwargs)
return sock_info.command(
self.__name,
command,
slave_ok,
read_preference,
codec_options,
check,
allowable_errors,
parse_write_concern_error=parse_write_concern_error)
def _gen_find_command(coll, spec, projection, skip, limit, batch_size,
options):
"""Generate a find command document."""
cmd = SON([('find', coll)])
if '$query' in spec:
cmd.update([(_MODIFIERS[key], val) for key, val in spec.items()])
else:
cmd['filter'] = spec
if projection:
cmd['projection'] = projection
if skip:
cmd['skip'] = skip
if limit:
cmd['limit'] = limit
if batch_size:
cmd['batchSize'] = batch_size
# XXX: Should the check for 1 be here?
if limit < 0 or limit == 1:
cmd['singleBatch'] = True
if options:
cmd.update([(opt, True) for opt, val in _OPTIONS.items()
if options & val])
return cmd
def compact_versions(
self,
start_version: int,
end_version: int,
destroy_proofs: Optional[bool] = None,
) -> CompactResponse:
"""Compacts all proofs, versions and documents in the db between two given versions,
deleting all data that only exists between the two versions.
See https://provendb.readme.io/docs/compact
:param start_version: The first version to compact from.
:type start_version: int
:param end_version: The last version to compact to.
:type end_version: int
:raises CompactException: pyproven exception when ProvenDB fails to compact between the two given versions.
:return: A dict-like object containing the number of deleted proofs, versions and documents.
:rtype: CompactResponse
"""
command_args = SON({"startVersion": start_version, "endVersion": end_version})
if destroy_proofs:
command_args.update({"destroyProofs": destroy_proofs})
try:
response = self.db.command("compact", command_args)
return CompactResponse(response)
except PyMongoError as err:
error_msg = extract_error_info(err)["errmsg"]
if (
error_msg
== "There must be a full proof above the range to be compacted"
):
raise CompactProofError(err)
else:
raise
def create_index(self, sort_fields, **kwargs):
def wrapper(result, name):
return name
if not isinstance(sort_fields, qf.sort):
raise TypeError("sort_fields must be an instance of filter.sort")
if "name" not in kwargs:
name = self._gen_index_name(sort_fields["orderby"])
else:
name = kwargs.pop("name")
key = SON()
for k, v in sort_fields["orderby"]:
key.update({k: v})
index = SON(dict(ns=str(self), name=name, key=key))
if "drop_dups" in kwargs:
kwargs["dropDups"] = kwargs.pop("drop_dups")
if "bucket_size" in kwargs:
kwargs["bucketSize"] = kwargs.pop("bucket_size")
index.update(kwargs)
d = self._database.system.indexes.insert(index, safe=True)
d.addCallback(wrapper, name)
return d
def find_and_modify(self, spec, document, upsert=False, manipulate=False,
safe=True, multi=False, callback=None, **kwargs):
if self._key_ in spec:
spec[MONGODB_ID] = spec[self._key_]
del spec[self._key_]
# add conditions delete_flag is not 1
spec[DELETE_FLAG] = {'$ne': '1'}
old_update_data = document.get("$set", None)
if old_update_data:
old_update_data["last_modify"] = int(time.time())
document['$set'] = old_update_data
from bson.son import SON
command = SON(
[('findAndModify', self._table_), ('query', spec), ('update', document), ('upsert', False), ('new', True)])
command.update(kwargs)
result = yield tornado.gen.Task(self.async_client.connection("$cmd", self._db_).find_one, command,
_must_use_master=True, _is_command=True)
flag = result[0][0]['value']
if flag and self.need_sync:
self.sync_update_data(spec, document)
callback(flag)
def find_and_modify(self, query={}, update=None, upsert=False, **kwargs):
def wrapper(result):
no_obj_error = "No matching object found"
if not result['ok']:
if result["errmsg"] == no_obj_error:
return None
else:
raise ValueError("Unexpected Error: %s" % (result, ))
return result.get('value')
if (not update and not kwargs.get('remove', None)):
raise ValueError("Must either update or remove")
if (update and kwargs.get('remove', None)):
raise ValueError("Can't do both update and remove")
cmd = SON([("findAndModify", self._collection_name)])
cmd.update(kwargs)
# No need to include empty args
if query:
cmd['query'] = query
if update:
cmd['update'] = update
if upsert:
cmd['upsert'] = upsert
d = self._database["$cmd"].find_one(cmd)
d.addCallback(wrapper)
return d
def __last_error(namespace, args):
"""Data to send to do a lastError.
"""
cmd = SON([("getlasterror", 1)])
cmd.update(args)
splitns = namespace.split(".", 1)
return query(0, splitns[0] + ".$cmd", 0, -1, cmd, None, DEFAULT_CODEC_OPTIONS)
def main():
persistence = mpl.MongoPersistenceLayer()
current_epoch = long(time.mktime(time.gmtime()))
users = persistence.find_records('user', {'access_token': {"$exists": True}, 'access_token_expires': {"$gt": current_epoch}},
{'_id':1, 'access_token':1, 'uname':1, 'friends':1})
for user in users:
client_longitude = -122.063796997
client_latitude = 37.2538986206
friend_ids = user['friends']
uid = user['_id']
me_friend_ids = [user['_id']] + friend_ids
logger.debug("uid: %s", uid)
logger.debug("num of associated users: %s", len(me_friend_ids))
logger.debug("associated users: %s", me_friend_ids)
import pdb;pdb.set_trace()
#me_friend_checkins = persistence.CHECKIN.find({"loc" : {"$near": [client_longitude, client_latitude]}, "author_uid": {"$in": me_friend_ids}})
#me_friend_checkins = persistence.CHECKIN.find({"loc" : {"$near": [client_longitude, client_latitude], "$maxDistance": 2.5}, "author_uid": {"$in": me_friend_ids}})
s = SON({'$near': [client_longitude, client_latitude]})
s.update({'$maxDistance': 0.5})
me_friend_checkins = persistence.CHECKIN.find({"loc" : s, "author_uid": {"$in": me_friend_ids}})
#me_friend_checkins = persistence.CHECKIN.find({"loc" : s}).limit(20)
count = me_friend_checkins.count()
logger.debug("length of no max distance checkins: %s", count)
for c in me_friend_checkins:
logger.debug("%s", c)
def create_index(self, sort_fields, **kwargs):
if not isinstance(sort_fields, qf.sort):
raise TypeError("TxMongo: sort_fields must be an instance of filter.sort")
if "name" not in kwargs:
name = self._gen_index_name(sort_fields["orderby"])
else:
name = kwargs.pop("name")
key = SON()
for k, v in sort_fields["orderby"]:
key.update({k: v})
index = SON(dict(
ns=str(self),
name=name,
key=key
))
if "drop_dups" in kwargs:
kwargs["dropDups"] = kwargs.pop("drop_dups")
if "bucket_size" in kwargs:
kwargs["bucketSize"] = kwargs.pop("bucket_size")
index.update(kwargs)
yield self._database.system.indexes.insert(index, safe=True, **kwargs)
defer.returnValue(name)
def _new_find_and_modify(self, filter, projection, sort, upsert=None,
return_document=ReturnDocument.BEFORE, **kwargs):
validate_is_mapping("filter", filter)
if not isinstance(return_document, bool):
raise ValueError("TxMongo: return_document must be ReturnDocument.BEFORE "
"or ReturnDocument.AFTER")
cmd = SON([("findAndModify", self._collection_name),
("query", filter),
("new", return_document)])
cmd.update(kwargs)
if projection is not None:
cmd["fields"] = self._normalize_fields_projection(projection)
if sort is not None:
cmd["sort"] = dict(sort["orderby"])
if upsert is not None:
validate_boolean("upsert", upsert)
cmd["upsert"] = upsert
no_obj_error = "No matching object found"
result = yield self._database.command(cmd, allowable_errors=[no_obj_error], **kwargs)
defer.returnValue(result.get("value"))
def mapping(data, dictionary):
"""Mapping a dictionary (SON) into a a MongoDB command.
The idea is: all values are keywords of a lambda functions dictionary,
each function take attribute as the only argument. Each lambda function
will generate a SON file consist of proper form of MongoDB operation
and a random values in designated place.
Note:
please notice that same keyword might represent different lambda
function in different type of operation.
E.g. Array.Num in update and insert represent different operator.
Example:
{'A1.B1': 'num_match'} + {'num_match' : lambda attr: {attr: self.values.randInt()},}
--> return {'A1.B1': 130416}
"""
res = SON()
for attr,cmd in data.items():
if isinstance(cmd, dict): # nested document: {'A1': {'B1': 'num_match'}}
res[attr] = mapping(cmd, dictionary)
else:
new = dictionary[cmd](attr)
assert(len(new) == 1)
k, v = new.items()[0]
if k in res: # duplicate attribute -> operators in update, e.g. $set, $push
res[k].update(v)
else:
res.update(new)
return res
def command(
self,
command,
value=1,
check=True,
allowable_errors=None,
codec_options=DEFAULT_CODEC_OPTIONS,
_deadline=None,
**kwargs
):
"""command(command, value=1, check=True, allowable_errors=None, codec_options=DEFAULT_CODEC_OPTIONS)"""
if isinstance(command, (bytes, unicode)):
command = SON([(command, value)])
options = kwargs.copy()
command.update(options)
def on_ok(response):
if check:
msg = "TxMongo: command {0} on namespace {1} failed with '%s'".format(repr(command), ns)
_check_command_response(response, msg, allowable_errors)
return response
ns = self["$cmd"].with_options(codec_options=codec_options)
return ns.find_one(command, _deadline=_deadline).addCallback(on_ok)
def __last_error(namespace, args):
"""Data to send to do a lastError.
"""
cmd = SON([("getlasterror", 1)])
cmd.update(args)
splitns = namespace.split('.', 1)
return query(0, splitns[0] + '.$cmd', 0, -1, cmd)
def create_index(self, sort_fields, **kwargs):
if not isinstance(sort_fields, qf.sort):
raise TypeError("sort_fields must be an instance of filter.sort")
if "name" not in kwargs:
name = self._gen_index_name(sort_fields["orderby"])
else:
name = kwargs.pop("name")
key = SON()
for k, v in sort_fields["orderby"]:
key.update({k: v})
index = SON(dict(
ns=str(self),
name=name,
key=key
))
if "drop_dups" in kwargs:
kwargs["dropDups"] = kwargs.pop("drop_dups")
if "bucket_size" in kwargs:
kwargs["bucketSize"] = kwargs.pop("bucket_size")
index.update(kwargs)
yield self._database.system.indexes.insert(index, safe=True)
defer.returnValue(name)
def _gen_find_command(coll, spec, projection, skip, limit, batch_size, options,
read_concern, collation=None):
"""Generate a find command document."""
cmd = SON([('find', coll)])
if '$query' in spec:
cmd.update([(_MODIFIERS[key], val) if key in _MODIFIERS else (key, val)
for key, val in spec.items()])
if '$explain' in cmd:
cmd.pop('$explain')
if '$readPreference' in cmd:
cmd.pop('$readPreference')
else:
cmd['filter'] = spec
if projection:
cmd['projection'] = projection
if skip:
cmd['skip'] = skip
if limit:
cmd['limit'] = abs(limit)
if limit < 0:
cmd['singleBatch'] = True
if batch_size:
cmd['batchSize'] = batch_size
if read_concern.level:
cmd['readConcern'] = read_concern.document
if collation:
cmd['collation'] = collation
if options:
cmd.update([(opt, True)
for opt, val in _OPTIONS.items()
if options & val])
return cmd
def __query_spec(self):
"""Get the spec to use for a query.
"""
operators = {}
if self.__ordering:
operators["$orderby"] = self.__ordering
if self.__explain:
operators["$explain"] = True
if self.__hint:
operators["$hint"] = self.__hint
if self.__snapshot:
operators["$snapshot"] = True
if self.__max_scan:
operators["$maxScan"] = self.__max_scan
if operators:
# Make a shallow copy so we can cleanly rewind or clone.
spec = self.__spec.copy()
if "$query" not in spec:
# $query has to come first
spec = SON({"$query": spec})
spec.update(operators)
return spec
# Have to wrap with $query if "query" is the first key.
# We can't just use $query anytime "query" is a key as
# that breaks commands like count and find_and_modify.
# Checking spec.keys()[0] covers the case that the spec
# was passed as an instance of SON or OrderedDict.
elif ("query" in self.__spec and
(len(self.__spec) == 1 or self.__spec.keys()[0] == "query")):
return SON({"$query": self.__spec})
return self.__spec
def command(self,
command,
value=1,
callback=None,
check=True,
allowable_errors=[],
**kwargs):
"""Issue a MongoDB command.
Send command `command` to the database and return the
response. If `command` is an instance of :class:`basestring`
then the command {`command`: `value`} will be sent. Otherwise,
`command` must be an instance of :class:`dict` and will be
sent as is.
Any additional keyword arguments will be added to the final
command document before it is sent.
For example, a command like ``{buildinfo: 1}`` can be sent
using:
>>> db.command("buildinfo")
For a command where the value matters, like ``{collstats:
collection_name}`` we can do:
>>> db.command("collstats", collection_name)
For commands that take additional arguments we can use
kwargs. So ``{filemd5: object_id, root: file_root}`` becomes:
>>> db.command("filemd5", object_id, root=file_root)
:Parameters:
- `command`: document representing the command to be issued,
or the name of the command (for simple commands only).
.. note:: the order of keys in the `command` document is
significant (the "verb" must come first), so commands
which require multiple keys (e.g. `findandmodify`)
should use an instance of :class:`~bson.son.SON` or
a string and kwargs instead of a Python `dict`.
- `value` (optional): value to use for the command verb when
`command` is passed as a string
- `**kwargs` (optional): additional keyword arguments will
be added to the command document before it is sent
.. mongodoc:: commands
"""
if isinstance(command, basestring):
command = SON([(command, value)])
command.update(kwargs)
self.connection("$cmd").find_one(command,
callback=callback,
_must_use_master=True,
_is_command=True)
def _command(self,
sock_info,
command,
slave_ok=False,
value=1,
check=True,
allowable_errors=None,
read_preference=ReadPreference.PRIMARY,
codec_options=DEFAULT_CODEC_OPTIONS,
write_concern=None,
parse_write_concern_error=False,
session=None,
**kwargs):
"""Internal command helper."""
if isinstance(command, str):
command = SON([(command, value)])
command.update(kwargs)
with self.__client._tmp_session(session) as s:
return sock_info.command(
self.__name,
command,
slave_ok,
read_preference,
codec_options,
check,
allowable_errors,
write_concern=write_concern,
parse_write_concern_error=parse_write_concern_error,
session=s,
client=self.__client)
def _gen_find_command(coll, spec, projection, skip, limit, batch_size, options):
"""Generate a find command document."""
cmd = SON([('find', coll)])
if '$query' in spec:
cmd.update([(_MODIFIERS[key], val) for key, val in spec.items()])
else:
cmd['filter'] = spec
if projection:
cmd['projection'] = projection
if skip:
cmd['skip'] = skip
if limit:
cmd['limit'] = limit
if batch_size:
cmd['batchSize'] = batch_size
# XXX: Should the check for 1 be here?
if limit < 0 or limit == 1:
cmd['singleBatch'] = True
if options:
cmd.update([(opt, True)
for opt, val in _OPTIONS.items()
if options & val])
return cmd
class SortStage(object):
key = '$sort'
def __init__(self):
self._sorts = SON()
def by(self, *args):
if not args:
raise RuntimeError('This method needs at least one argument')
if isinstance(args[0], list):
return self.by(*args[0])
if isinstance(args[0], tuple):
sort_by = SON(args)
else:
raise ValueError('The arguments to this method must be tuples')
self._sorts.update(sort_by)
return self
def sort(self, field, direction=1):
if isinstance(direction, basestring):
if direction.lower() == 'asc':
direction = 1
else:
direction = -1
self._sorts[field] = direction
return self
def build(self):
return {self.key: self._sorts}
def find_and_modify(self, query={}, update=None, upsert=False, **kwargs):
def wrapper(result):
no_obj_error = "No matching object found"
if not result['ok']:
if result["errmsg"] == no_obj_error:
return None
else:
raise ValueError("Unexpected Error: %s" % (result,))
return result.get('value')
if (not update and not kwargs.get('remove', None)):
raise ValueError("Must either update or remove")
if (update and kwargs.get('remove', None)):
raise ValueError("Can't do both update and remove")
cmd = SON([("findAndModify", self._collection_name)])
cmd.update(kwargs)
# No need to include empty args
if query:
cmd['query'] = query
if update:
cmd['update'] = update
if upsert:
cmd['upsert'] = upsert
d = self._database["$cmd"].find_one(cmd)
d.addCallback(wrapper)
return d
def _new_find_and_modify(self, filter, projection, sort, upsert=None,
return_document=ReturnDocument.BEFORE, **kwargs):
validate_is_mapping("filter", filter)
if not isinstance(return_document, bool):
raise ValueError("TxMongo: return_document must be ReturnDocument.BEFORE "
"or ReturnDocument.AFTER")
cmd = SON([("findAndModify", self._collection_name),
("query", filter),
("new", return_document)])
cmd.update(kwargs)
if projection is not None:
cmd["fields"] = self._normalize_fields_projection(projection)
if sort is not None:
cmd["sort"] = dict(sort["orderby"])
if upsert is not None:
validate_boolean("upsert", upsert)
cmd["upsert"] = upsert
no_obj_error = "No matching object found"
result = yield self._database.command(cmd, allowable_errors=[no_obj_error], **kwargs)
defer.returnValue(result.get("value"))
class SortStage(object):
key = '$sort'
def __init__(self):
self._sorts = SON()
def by(self, *args):
if not args:
raise RuntimeError('This method needs at least one argument')
if isinstance(args[0], list):
return self.by(*args[0])
if isinstance(args[0], tuple):
sort_by = SON(args)
else:
raise ValueError('The arguments to this method must be tuples')
self._sorts.update(sort_by)
return self
def sort(self, field, direction=1):
if isinstance(direction, basestring):
if direction.lower() == 'asc':
direction = 1
else:
direction = -1
self._sorts[field] = direction
return self
def build(self):
return {self.key: self._sorts}
def _gen_find_command(coll, spec, projection, skip, limit, batch_size, options, read_concern=DEFAULT_READ_CONCERN):
"""Generate a find command document."""
cmd = SON([("find", coll)])
if "$query" in spec:
cmd.update([(_MODIFIERS[key], val) if key in _MODIFIERS else (key, val) for key, val in spec.items()])
if "$explain" in cmd:
cmd.pop("$explain")
if "$readPreference" in cmd:
cmd.pop("$readPreference")
else:
cmd["filter"] = spec
if projection:
cmd["projection"] = projection
if skip:
cmd["skip"] = skip
if limit:
cmd["limit"] = abs(limit)
if limit < 0:
cmd["singleBatch"] = True
if batch_size:
cmd["batchSize"] = batch_size
if read_concern.level:
cmd["readConcern"] = read_concern.document
if options:
cmd.update([(opt, True) for opt, val in _OPTIONS.items() if options & val])
return cmd
def __last_error(namespace, args):
"""Data to send to do a lastError.
"""
cmd = SON([("getlasterror", 1)])
cmd.update(args)
splitns = namespace.split('.', 1)
return query(0, splitns[0] + '.$cmd', 0, -1, cmd)
def get_document_proof(
self,
collection: str,
filter: Dict[str, Any],
version: int,
proof_format: Optional[str] = None,
) -> GetDocumentProofResponse:
"""Filters documents in a collection and returns any proofs of those documents for a given version.
See: https://provendb.readme.io/docs/getdocumentproof
:param collection: The name of the collection to filter.
:type collection: str
:param filter: A mongodb filter that subsets the collection.
:type filter: Dict[str, Any]
:param version: The version number to fetch proofs for.
:type version: int
:param proof_format: The format of the proof, either 'binary' or 'json', defaults to "json"
:type proof_format: str
:raises GetDocumentProofException: [description]
:return: A dict-like object containing an array of document proof documents.
:rtype: GetDocumentProofResponse
"""
command_args = SON(
{
"collection": collection,
"filter": filter,
"version": version,
}
)
if proof_format:
command_args.update({"proofFormat": proof_format})
response = self.db.command("getDocumentProof", command_args)
return GetDocumentProofResponse(response)
def _gen_find_command(coll, spec, projection, skip, limit, batch_size,
options, read_concern=DEFAULT_READ_CONCERN):
"""Generate a find command document."""
cmd = SON([('find', coll)])
if '$query' in spec:
cmd.update([(_MODIFIERS[key], val) if key in _MODIFIERS else (key, val)
for key, val in spec.items()])
if '$explain' in cmd:
cmd.pop('$explain')
if '$readPreference' in cmd:
cmd.pop('$readPreference')
else:
cmd['filter'] = spec
if projection:
cmd['projection'] = projection
if skip:
cmd['skip'] = skip
if limit:
cmd['limit'] = abs(limit)
if limit < 0:
cmd['singleBatch'] = True
if batch_size:
cmd['batchSize'] = batch_size
if read_concern.level:
cmd['readConcern'] = read_concern.document
if options:
cmd.update([(opt, True)
for opt, val in _OPTIONS.items()
if options & val])
return cmd
def __query_spec(self):
"""Get the spec to use for a query.
"""
operators = {}
if self.__ordering:
operators["$orderby"] = self.__ordering
if self.__explain:
operators["$explain"] = True
if self.__hint:
operators["$hint"] = self.__hint
if self.__snapshot:
operators["$snapshot"] = True
if self.__max_scan:
operators["$maxScan"] = self.__max_scan
if operators:
# Make a shallow copy so we can cleanly rewind or clone.
spec = self.__spec.copy()
if "$query" not in spec:
# $query has to come first
spec = SON({"$query": spec})
spec.update(operators)
return spec
# Have to wrap with $query if "query" is the first key.
# We can't just use $query anytime "query" is a key as
# that breaks commands like count and find_and_modify.
# Checking spec.keys()[0] covers the case that the spec
# was passed as an instance of SON or OrderedDict.
elif ("query" in self.__spec
and (len(self.__spec) == 1 or self.__spec.keys()[0] == "query")):
return SON({"$query": self.__spec})
return self.__spec
def _command(self, command, value=1,
check=True, allowable_errors=None,
uuid_subtype=OLD_UUID_SUBTYPE, compile_re=True,
read_preference=None, **kwargs):
"""Internal command helper.
"""
if isinstance(command, basestring):
command_name = command.lower()
command = SON([(command, value)])
else:
command_name = command.keys()[0].lower()
as_class = kwargs.pop('as_class', None)
fields = kwargs.pop('fields', None)
if fields is not None and not isinstance(fields, dict):
fields = helpers._fields_list_to_dict(fields)
command.update(kwargs)
orig = mode = read_preference or self.read_preference
if command_name not in SECONDARY_OK_COMMANDS:
mode = ReadPreference.PRIMARY
# Special-case: mapreduce can go to secondaries only if inline
elif command_name == 'mapreduce':
out = command.get('out')
if not isinstance(out, dict) or not out.get('inline'):
mode = ReadPreference.PRIMARY
# Special-case: aggregate with $out cannot go to secondaries.
elif command_name == 'aggregate':
for stage in command.get('pipeline', []):
if '$out' in stage:
mode = ReadPreference.PRIMARY
break
# Warn if mode will override read_preference.
if mode != orig:
warnings.warn("%s does not support %s read preference "
"and will be routed to the primary instead." %
(command_name, orig.name), UserWarning)
cursor = self["$cmd"].find(command,
fields=fields,
limit=-1,
as_class=as_class,
read_preference=mode,
compile_re=compile_re,
_uuid_subtype=uuid_subtype)
for doc in cursor:
result = doc
if check:
msg = "command %s failed: %%s" % repr(command).replace("%", "%%")
helpers._check_command_response(result, self.connection.disconnect,
msg, allowable_errors)
return result, cursor.conn_id
def _create_user(authdb, user, pwd=None, roles=None, **kwargs):
cmd = SON([('createUser', user)])
# X509 doesn't use a password
if pwd:
cmd['pwd'] = pwd
cmd['roles'] = roles or ['root']
cmd.update(**kwargs)
return authdb.command(cmd)
def transform_incoming(self, son, collection):
"""Move _id to the front if it's there.
"""
if not "_id" in son:
return son
transformed = SON({"_id": son["_id"]})
transformed.update(son)
return transformed
def transform_incoming(self, son, collection):
"""Move _id to the front if it's there.
"""
if not "_id" in son:
return son
transformed = SON({"_id": son["_id"]})
transformed.update(son)
return transformed
def _create_user(authdb, user, pwd=None, roles=None, **kwargs):
cmd = SON([('createUser', user)])
# X509 doesn't use a password
if pwd:
cmd['pwd'] = pwd
cmd['roles'] = roles or ['root']
cmd.update(**kwargs)
return authdb.command(cmd)
def fail_point(self, command_args):
cmd_on = SON([('configureFailPoint', 'failCommand')])
cmd_on.update(command_args)
client_context.client.admin.command(cmd_on)
try:
yield
finally:
client_context.client.admin.command(
'configureFailPoint', cmd_on['configureFailPoint'], mode='off')
def __query_spec(self):
"""Get the spec to use for a query.
"""
operators = {}
if self.__ordering:
operators["$orderby"] = self.__ordering
if self.__explain:
operators["$explain"] = True
if self.__hint:
operators["$hint"] = self.__hint
if self.__let:
operators["let"] = self.__let
if self.__comment:
operators["$comment"] = self.__comment
if self.__max_scan:
operators["$maxScan"] = self.__max_scan
if self.__max_time_ms is not None:
operators["$maxTimeMS"] = self.__max_time_ms
if self.__max:
operators["$max"] = self.__max
if self.__min:
operators["$min"] = self.__min
if self.__return_key is not None:
operators["$returnKey"] = self.__return_key
if self.__show_record_id is not None:
# This is upgraded to showRecordId for MongoDB 3.2+ "find" command.
operators["$showDiskLoc"] = self.__show_record_id
if self.__snapshot is not None:
operators["$snapshot"] = self.__snapshot
if operators:
# Make a shallow copy so we can cleanly rewind or clone.
spec = self.__spec.copy()
# Allow-listed commands must be wrapped in $query.
if "$query" not in spec:
# $query has to come first
spec = SON([("$query", spec)])
if not isinstance(spec, SON):
# Ensure the spec is SON. As order is important this will
# ensure its set before merging in any extra operators.
spec = SON(spec)
spec.update(operators)
return spec
# Have to wrap with $query if "query" is the first key.
# We can't just use $query anytime "query" is a key as
# that breaks commands like count and find_and_modify.
# Checking spec.keys()[0] covers the case that the spec
# was passed as an instance of SON or OrderedDict.
elif ("query" in self.__spec and
(len(self.__spec) == 1 or next(iter(self.__spec)) == "query")):
return SON({"$query": self.__spec})
return self.__spec
def repl_set_step_down(client, **kwargs):
"""Run replSetStepDown, first unfreezing a secondary with replSetFreeze."""
cmd = SON([('replSetStepDown', 1)])
cmd.update(kwargs)
# Unfreeze a secondary to ensure a speedy election.
client.admin.command('replSetFreeze',
0,
read_preference=ReadPreference.SECONDARY)
client.admin.command(cmd)
def as_doc(self):
"""Get the SON document representation of this DBRef.
Generally not needed by application developers
"""
doc = SON([("$ref", self.collection), ("$id", self.id)])
if self.database is not None:
doc["$db"] = self.database
doc.update(self.__kwargs)
return doc
def as_doc(self):
"""Get the SON document representation of this DBRef.
Generally not needed by application developers
"""
doc = SON([("$ref", self.collection), ("$id", self.id)])
if self.database is not None:
doc["$db"] = self.database
doc.update(self.__kwargs)
return doc
def __query_spec(self):
"""Get the spec to use for a query.
"""
operators = self.__modifiers.copy()
if self.__ordering:
operators["$orderby"] = self.__ordering
if self.__explain:
operators["$explain"] = True
if self.__hint:
operators["$hint"] = self.__hint
if self.__comment:
operators["$comment"] = self.__comment
if self.__max_scan:
operators["$maxScan"] = self.__max_scan
if self.__max_time_ms is not None:
operators["$maxTimeMS"] = self.__max_time_ms
if self.__max:
operators["$max"] = self.__max
if self.__min:
operators["$min"] = self.__min
if self.__return_key:
operators["$returnKey"] = self.__return_key
if self.__show_record_id:
# This is upgraded to showRecordId for MongoDB 3.2+ "find" command.
operators["$showDiskLoc"] = self.__show_record_id
if self.__snapshot:
operators["$snapshot"] = self.__snapshot
if operators:
# Make a shallow copy so we can cleanly rewind or clone.
spec = self.__spec.copy()
# White-listed commands must be wrapped in $query.
if "$query" not in spec:
# $query has to come first
spec = SON([("$query", spec)])
if not isinstance(spec, SON):
# Ensure the spec is SON. As order is important this will
# ensure its set before merging in any extra operators.
spec = SON(spec)
spec.update(operators)
return spec
# Have to wrap with $query if "query" is the first key.
# We can't just use $query anytime "query" is a key as
# that breaks commands like count and find_and_modify.
# Checking spec.keys()[0] covers the case that the spec
# was passed as an instance of SON or OrderedDict.
elif ("query" in self.__spec and
(len(self.__spec) == 1 or
next(iter(self.__spec)) == "query")):
return SON({"$query": self.__spec})
return self.__spec
def command(self, command, value=1, callback=None,
check=True, allowable_errors=[], **kwargs):
"""Issue a MongoDB command.
Send command `command` to the database and return the
response. If `command` is an instance of :class:`basestring`
then the command {`command`: `value`} will be sent. Otherwise,
`command` must be an instance of :class:`dict` and will be
sent as is.
Any additional keyword arguments will be added to the final
command document before it is sent.
For example, a command like ``{buildinfo: 1}`` can be sent
using:
>>> db.command("buildinfo")
For a command where the value matters, like ``{collstats:
collection_name}`` we can do:
>>> db.command("collstats", collection_name)
For commands that take additional arguments we can use
kwargs. So ``{filemd5: object_id, root: file_root}`` becomes:
>>> db.command("filemd5", object_id, root=file_root)
:Parameters:
- `command`: document representing the command to be issued,
or the name of the command (for simple commands only).
.. note:: the order of keys in the `command` document is
significant (the "verb" must come first), so commands
which require multiple keys (e.g. `findandmodify`)
should use an instance of :class:`~bson.son.SON` or
a string and kwargs instead of a Python `dict`.
- `value` (optional): value to use for the command verb when
`command` is passed as a string
- `**kwargs` (optional): additional keyword arguments will
be added to the command document before it is sent
.. mongodoc:: commands
"""
if isinstance(command, str):
command = SON([(command, value)])
command.update(kwargs)
self.connection("$cmd").find_one(command,callback=callback,
_must_use_master=True,
_is_command=True)
def map_reduce(self, map, reduce, full_response=False, **kwargs):
def wrapper(result, full_response):
if full_response:
return result
return result.get("result")
cmd = SON([("mapreduce", self._collection_name),
("map", map), ("reduce", reduce)])
cmd.update(**kwargs)
d = self._database["$cmd"].find_one(cmd)
d.addCallback(wrapper, full_response)
return d
def map_reduce(self, map, reduce, full_response=False, **kwargs):
def wrapper(result, full_response):
if full_response:
return result
return result.get("result")
cmd = SON([("mapreduce", self._collection_name), ("map", map),
("reduce", reduce)])
cmd.update(**kwargs)
d = self._database["$cmd"].find_one(cmd)
d.addCallback(wrapper, full_response)
return d
def custom_parallel_scan(collection, num_cursors, **kwargs):
cmd = SON([('parallelCollectionScan', collection._Collection__name),
('numCursors', num_cursors)])
cmd.update(kwargs)
with collection._socket_for_reads() as (sock_info, slave_ok):
result = collection._command(sock_info,
cmd,
slave_ok,
read_concern=collection.read_concern)
# return [cursor['cursor'] for cursor in result['cursors']]
return result['cursors']
def command(self, command, value=1, check=True, allowable_errors=None, **kwargs):
if isinstance(command, basestring):
command = SON([(command, value)])
command.update(kwargs)
ns = self["$cmd"]
response = yield ns.find_one(command)
if check:
msg = "command {0} on namespace {1} failed: %s".format(repr(command), ns)
_check_command_response(response, msg, allowable_errors)
defer.returnValue(response)
def list_versions(
self,
start_date: Optional[datetime.datetime] = None,
end_date: Optional[datetime.datetime] = None,
limit: Optional[int] = None,
sort_direction: Optional[int] = None,
) -> ListVersionsResponse:
"""Retrieves a list of versions given a search parameter.
See https://provendb.readme.io/docs/listversions
:param start_date: Specifies first date to retrieve versions, defaults to 24 hours from now
:type start_date: Optional[datetime.datetime]
:param end_date: Last date to retrieve documents, defaults to now
:type end_date: Optional[datetime.datetime]
:param limit: Number of version documents to retrieve, defaults to 10
:type limit: Optional[int], optional
:param sort_direction: -1 to retrieve versions in descending order, 1 ascending order. Defaults to -1.
:type sort_direction: Optional[int], optional
:raises ListVersionException: pyproven exception when fails to retrieve version list.
:return: A dict-like object representing the ProvenDB response document.
:rtype: ListVersionsResponse
"""
command_args = SON()
if start_date:
command_args.update({"startDate": start_date})
if end_date:
command_args.update({"endDate": end_date})
if limit:
command_args.update({"limit": limit})
if sort_direction:
command_args.update({"sortDirection": sort_direction})
response = self.db.command({"listVersions": command_args})
return ListVersionsResponse(response)
def createDBUsers(self):
conn = self.__get_connection()
yield conn["admin"].authenticate(self.ua_login, self.ua_password)
create_user = SON({"createUser": self.login1})
create_user.update({"pwd": self.password1, "roles": [{"role": "readWrite",
"db": self.db1}]})
yield conn[self.db1]["$cmd"].find_one(create_user)
create_user = SON({"createUser": self.login2})
create_user.update({"pwd": self.password2, "roles": [{"role": "readWrite",
"db": self.db2}]})
yield conn[self.db2]["$cmd"].find_one(create_user)
yield conn.disconnect()
def _on_get_nonce(self, response, error=None):
if error:
raise AuthenticationError(error)
nonce = response['data'][0]['nonce']
key = hashlib.md5(nonce + self._dbuser + hashlib.md5(self._dbuser + ":mongo:" + self._dbpass).hexdigest()).hexdigest()
command = SON([('authenticate', 1)])
command.update({'user' : self._dbuser, 'nonce' : nonce, 'key' : key})
self.conn.send_message(
message.query(0,
"%s.$cmd" % self._dbname,
0,
1,
command,
SON({})),callback=self._on_authenticate)
def command(self, command, value=1, check=True, allowable_errors=None, **kwargs):
if isinstance(command, (bytes, unicode)):
command = SON([(command, value)])
options = kwargs.copy()
options.pop("_deadline", None)
command.update(options)
ns = self["$cmd"]
response = yield ns.find_one(command, **kwargs)
if check:
msg = "TxMongo: command {0} on namespace {1} failed with '%s'".format(repr(command), ns)
_check_command_response(response, msg, allowable_errors)
defer.returnValue(response)
def count(self, query=None, callback=None):
command = SON({
"count": self.collection.__collection__
})
if query:
command.update({'query': query})
result, error = yield gen.Task(Database().command, command)
total = 0
if result and len(result) > 0 and 'n' in result:
total = int(result['n'])
callback(total)
async def list_collections(self,
*,
session: Optional[AsyncClientSession] = None,
query: Optional[MutableMapping[str,
Any]] = None,
**kwargs: Any) -> AsyncCommandCursor:
cmd = SON([("listCollections", 1)])
cmd.update(query, **kwargs)
res: MutableMapping[str, Any] = await util.run_sync(
self.dispatch._retryable_read_command, # skipcq: PYL-W0212
cmd,
session=session.dispatch if session else session)
return AsyncCommandCursor(
CommandCursor(self["$cmd"], res["cursor"], None))
def _command(self, sock_info, command, slave_ok=False, value=1, check=True,
allowable_errors=None, read_preference=ReadPreference.PRIMARY,
codec_options=CodecOptions(), **kwargs):
"""Internal command helper."""
if isinstance(command, string_type):
command = SON([(command, value)])
command.update(kwargs)
return sock_info.command(self.__name,
command,
slave_ok,
read_preference,
codec_options,
check,
allowable_errors)
def _command(self, sock_info, command, slave_ok=False, value=1, check=True,
allowable_errors=None, read_preference=ReadPreference.PRIMARY,
codec_options=DEFAULT_CODEC_OPTIONS, **kwargs):
"""Internal command helper."""
if isinstance(command, string_type):
command = SON([(command, value)])
command.update(kwargs)
return sock_info.command(self.__name,
command,
slave_ok,
read_preference,
codec_options,
check,
allowable_errors)
def __query_spec(self) -> SON:
"""Get the spec to use for a query.
"""
operators = self.__modifiers
if self.__ordering:
operators['$orderby'] = self.__ordering
if self.__explain:
operators['$explain'] = True
if self.__hint:
operators['$hint'] = self.__hint
if self.__comment:
operators['$comment'] = self.__comment
if self.__max_scan:
operators['$maxScan'] = self.__max_scan
if self.__max_time_ms is not None:
operators['$maxTimeMS'] = self.__max_time_ms
if self.__max:
operators['$max'] = self.__max
if self.__min:
operators['$min'] = self.__min
if operators:
# Make a shallow copy so we can cleanly rewind or clone.
spec = self.__spec.copy()
# White-listed commands must be wrapped in $query.
if '$query' not in spec:
# $query has to come first
spec = SON([('$query', spec)])
if not isinstance(spec, SON):
# Ensure the spec is SON. As order is important this will
# ensure its set before merging in any extra operators.
spec = SON(spec)
spec.update(operators)
return spec
# Have to wrap with $query if 'query' is the first key.
# We can't just use $query anytime 'query' is a key as
# that breaks commands like count and find_and_modify.
# Checking spec.keys()[0] covers the case that the spec
# was passed as an instance of SON or OrderedDict.
elif ('query' in self.__spec and
(len(self.__spec) == 1 or
next(iter(self.__spec)) == 'query')):
return SON({'$query': self.__spec})
return self.__spec
def distinct(self, key, callback, query=None):
"""Returns a list of distinct values for the given
key across collection"""
command = SON({
"distinct": self.collection.__collection__,
"key": key,
})
if query:
command.update({'query': query})
result, error = yield gen.Task(Database().command, command)
if result and result['ok']:
callback(result['values'])
else:
callback(None)
def command(self, command, value=1, check=True, allowable_errors=None,
codec_options=DEFAULT_CODEC_OPTIONS, _deadline=None, **kwargs):
"""command(command, value=1, check=True, allowable_errors=None, codec_options=DEFAULT_CODEC_OPTIONS)"""
if isinstance(command, (bytes, unicode)):
command = SON([(command, value)])
options = kwargs.copy()
command.update(options)
def on_ok(response):
if check:
msg = "TxMongo: command {0} on namespace {1} failed with '%s'".format(repr(command), ns)
_check_command_response(response, msg, allowable_errors)
return response
ns = self["$cmd"].with_options(codec_options=codec_options)
return ns.find_one(command, _deadline=_deadline).addCallback(on_ok)
def _gen_find_command(
coll,
spec,
projection,
skip,
limit,
batch_size,
options,
read_concern,
collation=None,
session=None,
allow_disk_use=None,
):
"""Generate a find command document."""
cmd = SON([("find", coll)])
if "$query" in spec:
cmd.update([(_MODIFIERS[key], val) if key in _MODIFIERS else (key, val)
for key, val in spec.items()])
if "$explain" in cmd:
cmd.pop("$explain")
if "$readPreference" in cmd:
cmd.pop("$readPreference")
else:
cmd["filter"] = spec
if projection:
cmd["projection"] = projection
if skip:
cmd["skip"] = skip
if limit:
cmd["limit"] = abs(limit)
if limit < 0:
cmd["singleBatch"] = True
if batch_size:
cmd["batchSize"] = batch_size
if read_concern.level and not (session and session.in_transaction):
cmd["readConcern"] = read_concern.document
if collation:
cmd["collation"] = collation
if allow_disk_use is not None:
cmd["allowDiskUse"] = allow_disk_use
if options:
cmd.update([(opt, True) for opt, val in _OPTIONS.items()
if options & val])
return cmd
def _command(self, sock_info, command, slave_ok=False, value=1, check=True,
allowable_errors=None, read_preference=ReadPreference.PRIMARY,
codec_options=CodecOptions(), **kwargs):
"""Internal command helper."""
if isinstance(command, string_type):
command = SON([(command, value)])
command.update(kwargs)
result = sock_info.command(self.__name,
command,
slave_ok,
read_preference,
codec_options,
check,
allowable_errors)
# TODO: don't return address, caller knows it
return result, sock_info.address
def _list_collections(self, sock_info, slave_okay, session,
read_preference, **kwargs):
"""Internal listCollections helper."""
coll = self.get_collection("$cmd", read_preference=read_preference)
if sock_info.max_wire_version > 2:
cmd = SON([("listCollections", 1), ("cursor", {})])
cmd.update(kwargs)
with self.__client._tmp_session(session,
close=False) as tmp_session:
cursor = self._command(sock_info,
cmd,
slave_okay,
read_preference=read_preference,
session=tmp_session)["cursor"]
cmd_cursor = CommandCursor(coll,
cursor,
sock_info.address,
session=tmp_session,
explicit_session=session
is not None)
else:
match = _INDEX_REGEX
if "filter" in kwargs:
match = {"$and": [_INDEX_REGEX, kwargs["filter"]]}
dblen = len(self.name.encode("utf8") + b".")
pipeline = [{
"$project": {
"name": {
"$substr": ["$name", dblen, -1]
},
"options": 1
}
}, {
"$match": match
}]
cmd = SON([("aggregate", "system.namespaces"),
("pipeline", pipeline),
("cursor", kwargs.get("cursor", {}))])
cursor = self._command(sock_info, cmd, slave_okay)["cursor"]
cmd_cursor = CommandCursor(coll, cursor, sock_info.address)
cmd_cursor._maybe_pin_connection(sock_info)
return cmd_cursor
def createUserAdmin(self):
conn = self.__get_connection()
try:
create_user = SON({"createUser": self.ua_login})
create_user.update({"pwd": self.ua_password, "roles": [{"role": "userAdminAnyDatabase",
"db": "admin"}]})
r = yield conn["admin"]["$cmd"].find_one(create_user)
try:
# This should fail if authentication enabled in MongoDB since
# we've created user but didn't authenticated
yield conn[self.db1][self.coll].find_one()
yield conn["admin"]["$cmd"].find_one({"dropUser": self.ua_login})
raise unittest.SkipTest("Authentication tests require authorization enabled "
"in MongoDB configuration file")
except OperationFailure:
pass
finally:
yield conn.disconnect()
def create_collection(self, name, options={}):
def wrapper(result, deferred, collection):
if result.get("ok", 0.0):
deferred.callback(collection)
else:
deferred.errback(RuntimeError(result.get("errmsg", "unknown error")))
deferred = defer.Deferred()
collection = Collection(self, name)
if options:
if "size" in options:
options["size"] = float(options["size"])
command = SON({"create": name})
command.update(options)
d = self["$cmd"].find_one(command)
d.addCallback(wrapper, deferred, collection)
else:
deferred.callback(collection)
return deferred
