def find_one(self,
spec_or_object_id=None,
fields=None,
_sock=None,
_must_use_master=False,
_is_command=False):
"""Get a single object from the database.
Raises TypeError if the argument is of an improper type. Returns a
single SON object, or None if no result is found.
:Parameters:
- `spec_or_object_id` (optional): a SON object specifying elements
which must be present for a document to be returned OR an instance
of ObjectId to be used as the value for an _id query
- `fields` (optional): a list of field names that should be included
in the returned document ("_id" will always be included)
"""
spec = spec_or_object_id
if spec is None:
spec = SON()
if isinstance(spec, ObjectId):
spec = SON({"_id": spec})
for result in self.find(spec,
limit=-1,
fields=fields,
_sock=_sock,
_must_use_master=_must_use_master,
_is_command=_is_command):
return result
return None
def _touch_query(self):
if self._query_additions:
spec = SON({'query': self.spec})
for k, v in self._query_additions:
if k == 'sort':
ordering = spec.setdefault('orderby', SON())
ordering.update(v)
self.spec = spec
def next_big_id(abbr, letter, collection):
query = SON([('_id', abbr)])
update = SON([('$inc', SON([('seq', 1)]))])
seq = db.command(
SON([('findandmodify', collection),
('query', query), ('update', update), ('new', True),
('upsert', True)]))['value']['seq']
return "%s%s%08d" % (abbr.upper(), letter, seq)
def get_next_id(self):
# Generate a new sequential ID for the vote
query = SON([('_id', self.state)])
update = SON([('$inc', SON([('seq', 1)]))])
seq = db.command(
SON([('findandmodify', 'vote_ids'), ('query', query),
('update', update), ('new', True),
('upsert', True)]))['value']['seq']
return "%sV%08d" % (self.state.upper(), seq)
def test_from_xml(self):
smorgasbord = """
<twonk>
<meta/>
<doc>
<oid name="_id">285a664923b5fcd8ec000000</oid>
<int name="the_answer">42</int>
<string name="b">foo</string>
<boolean name="c">true</boolean>
<number name="pi">3.14159265358979</number>
<array name="an_array">
<string name="0">x</string>
<string name="1">y</string>
<string name="2">z</string>
<doc name="3">
<string name="subobject">yup</string>
</doc>
</array>
<date name="now">123144452057</date>
<ref name="dbref">
<ns>namespace</ns>
<oid>ca5c67496c01d896f7010000</oid>
</ref>
<regex name="regex">
<pattern>foobar</pattern>
<options>i</options>
</regex>
<code name="$where">this is code</code>
<null name="mynull"/>
</doc>
</twonk>
"""
self.assertEqual(
SON.from_xml(smorgasbord),
SON([
(u"_id",
ObjectId("\x28\x5A\x66\x49\x23\xB5\xFC"
"\xD8\xEC\x00\x00\x00")),
(u"the_answer", 42),
(u"b", u"foo"),
(u"c", True),
(u"pi", 3.14159265358979),
(u"an_array",
[u"x", u"y", u"z",
SON([(u"subobject", u"yup")])]),
(u"now", datetime.datetime(1973, 11, 26, 6, 47, 32, 57000)),
(u"dbref",
DBRef(
"namespace",
ObjectId("\xCA\x5C\x67\x49\x6C\x01"
"\xD8\x96\xF7\x01\x00\x00"))),
(u"regex", re.compile(u"foobar", re.IGNORECASE)),
(u"$where", "this is code"),
(u"mynull", None),
]))
def test_ordered_dict(self):
a = SON()
a["hello"] = "world"
a["mike"] = "awesome"
a["hello_"] = "mike"
self.assertEqual(a.items(), [("hello", "world"), ("mike", "awesome"),
("hello_", "mike")])
b = SON({"hello": "world"})
self.assertEqual(b["hello"], "world")
self.assertRaises(KeyError, lambda: b["goodbye"])
def getStatuses():
for i in range(4): # 30 secs
if i != 0:
sleep(10)
print_("fetching serverStatus #%s" % (i + 1))
stat = admin.command('serverStatus')
statuses.append(stat)
status_times.append(str(datetime.datetime.now()))
if i == 0:
status_diffs.append(None)
else:
last = statuses[i - 1]
time = (stat['uptime'] - last['uptime']) * 1.0 # just to be safe
diff = SON()
diff['ops_per_sec'] = SON()
diff['ops_per_sec']['insert'] = (
stat['opcounters']['insert'] -
last['opcounters']['insert']) / time
diff['ops_per_sec']['query'] = (stat['opcounters']['query'] -
last['opcounters']['query']) / time
diff['ops_per_sec']['update'] = (
stat['opcounters']['update'] -
last['opcounters']['update']) / time
diff['ops_per_sec']['delete'] = (
stat['opcounters']['delete'] -
last['opcounters']['delete']) / time
diff['ops_per_sec']['getmore'] = (
stat['opcounters']['getmore'] -
last['opcounters']['getmore']) / time
diff['ops_per_sec']['command'] = (
stat['opcounters']['command'] -
last['opcounters']['command']) / time
diff['btree_miss_per_sec'] = (
stat['indexCounters']['btree']['misses'] -
last['indexCounters']['btree']['misses']) / time
try:
diff['heap_change'] = stat['extra_info'][
'heap_usage_bytes'] - last['extra_info']['heap_usage_bytes']
diff['pflts_per_sec'] = (
stat['extra_info']['page_faults'] -
last['extra_info']['page_faults']) / time
except KeyError:
pass # these fields are only on linux
status_diffs.append(diff)
def _insert_with_id(event):
query = SON([('_id', event['state'])])
update = SON([('$inc', SON([('seq', 1)]))])
seq = db.command(
SON([('findandmodify', 'event_ids'), ('query', query),
('update', update), ('new', True),
('upsert', True)]))['value']['seq']
id = "%sE%08d" % (event['state'].upper(), seq)
logging.info("Saving as %s" % id)
event['_id'] = id
db.events.save(event, safe=True)
return id
def put_document(doc, content_type, metadata):
# Generate a new sequential ID for the document
query = SON([('_id', metadata['bill']['state'])])
update = SON([('$inc', SON([('seq', 1)]))])
seq = db.command(
SON([('findandmodify', 'doc_ids'),
('query', query), ('update', update), ('new', True),
('upsert', True)]))['value']['seq']
id = "%sD%08d" % (metadata['bill']['state'].upper(), seq)
logging.info("Saving as %s" % id)
fs.put(doc, _id=id, content_type=content_type, metadata=metadata)
return id
def rename(self, new_name):
"""Rename this collection.
If operating in auth mode, client must be authorized as an
admin to perform this operation. Raises :class:`TypeError` if
`new_name` is not an instance of :class:`basestring`. Raises
:class:`~pymongo.errors.InvalidName` if `new_name` is not a
valid collection name.
:Parameters:
- `new_name`: new name for this collection
"""
if not isinstance(new_name, basestring):
raise TypeError("new_name must be an instance of basestring")
if not new_name or ".." in new_name:
raise InvalidName("collection names cannot be empty")
if "$" in new_name:
raise InvalidName("collection names must not contain '$'")
if new_name[0] == "." or new_name[-1] == ".":
raise InvalidName("collecion names must not start or end with '.'")
rename_command = SON([("renameCollection", self.__full_name),
("to",
"%s.%s" % (self.__database.name, new_name))])
self.__database.connection.admin.command(rename_command)
def drop_index(self, index_or_name):
"""Drops the specified index on this collection.
Can be used on non-existant collections or collections with no indexes.
Raises OperationFailure on an error. `index_or_name` can be either an
index name (as returned by `create_index`), or an index specifier (as
passed to `create_index`). An index specifier should be a list of (key,
direction) pairs. Raises TypeError if index is not an instance of (str,
unicode, list).
:Parameters:
- `index_or_name`: index (or name of index) to drop
"""
name = index_or_name
if isinstance(index_or_name, list):
name = self._gen_index_name(index_or_name)
if not isinstance(name, basestring):
raise TypeError("index_or_name must be an index name or list")
self.__database.connection._purge_index(self.__database.name,
self.__name, name)
self.__database.command(
SON([("deleteIndexes", self.__name), ("index", name)]),
["ns not found"])
def test_save_find_one(self):
db = Database(self.connection, "pymongo_test")
db.test.remove({})
a_doc = SON({"hello": u"world"})
a_key = db.test.save(a_doc)
self.assert_(isinstance(a_doc["_id"], ObjectId))
self.assertEqual(a_doc["_id"], a_key)
self.assertEqual(a_doc, db.test.find_one({"_id": a_doc["_id"]}))
self.assertEqual(a_doc, db.test.find_one(a_key))
self.assertEqual(None, db.test.find_one(ObjectId()))
self.assertEqual(a_doc, db.test.find_one({"hello": u"world"}))
self.assertEqual(None, db.test.find_one({"hello": u"test"}))
b = db.test.find_one()
b["hello"] = u"mike"
db.test.save(b)
self.assertNotEqual(a_doc, db.test.find_one(a_key))
self.assertEqual(b, db.test.find_one(a_key))
self.assertEqual(b, db.test.find_one())
count = 0
for _ in db.test.find():
count += 1
self.assertEqual(count, 1)
def check_for_pos_dupes_via_geoloc(attrs, collection_name=None):
if attrs['geometry_type'] == "Point":
ll = {
'$near': [
float(attrs['geometry_centroid'][0]),
float(attrs['geometry_centroid'][1])
]
}
cc = ClassifierCategories.objects.get(
slug=attrs['classifiers']['category'])
md = {'$maxDistance': int(cc.duplicate_distance_tolerance)}
q = SON(ll)
q.update(md)
gq = {
'geometry_centroid': q,
'classifiers.category': attrs['classifiers']['category']
}
x = query_mongo_db(gq, collection_name=collection_name)
if x.has_key('features'):
if len(x['features']) > 0:
attrs['possible_duplicate'] = True
return attrs
def _index_document(index_list):
"""Helper to generate an index specifying document.
Takes a list of (key, direction) pairs.
"""
if isinstance(index_list, dict):
raise TypeError("passing a dict to sort/create_index/hint is not "
"allowed - use a list of tuples instead. did you "
"mean %r?" % list(index_list.iteritems()))
elif not isinstance(index_list, list):
raise TypeError("must use a list of (key, direction) pairs, "
"not: %r" % index_list)
if not len(index_list):
raise ValueError("key_or_list must not be the empty list")
index = SON()
for (key, value) in index_list:
if not isinstance(key, basestring):
raise TypeError("first item in each key pair must be a string")
if not isinstance(value, int):
raise TypeError(
"second item in each key pair must be ASCENDING or "
"DESCENDING")
index[key] = value
return index
def sort(self, name, direction):
if self.retrieved:
raise ValueError("can't sort an already started cursor")
key = SON()
key[name] = direction
self._query_additions.append(('sort', key))
return self
def test_from_then_to_dict(self):
def helper(dict):
self.assertEqual(dict, (BSON.from_dict(dict)).to_dict())
helper({})
helper({"test": u"hello"})
self.assert_(
isinstance(
BSON.from_dict({
"hello": "world"
}).to_dict()["hello"], types.UnicodeType))
helper({"mike": -10120})
helper({"long": long(10)})
helper({"really big long": 2147483648})
helper({u"hello": 0.0013109})
helper({"something": True})
helper({"false": False})
helper({"an array": [1, True, 3.8, u"world"]})
helper({"an object": {"test": u"something"}})
helper({"a binary": Binary("test", 100)})
helper({"a binary": Binary("test", 128)})
helper({"a binary": Binary("test", 254)})
helper({"another binary": Binary("test")})
helper(SON([(u'test dst', datetime.datetime(1993, 4, 4, 2))]))
helper({"big float": float(10000000000)})
def from_then_to_dict(dict):
return dict == (BSON.from_dict(dict)).to_dict()
qcheck.check_unittest(self, from_then_to_dict,
qcheck.gen_mongo_dict(3))
def count(self, with_limit_and_skip=False):
"""Get the size of the results set for this query.
Returns the number of documents in the results set for this query. Does
not take :meth:`limit` and :meth:`skip` into account by default - set
`with_limit_and_skip` to ``True`` if that is the desired behavior.
Raises :class:`~pymongo.errors.OperationFailure` on a database error.
:Parameters:
- `with_limit_and_skip` (optional): take any :meth:`limit` or
:meth:`skip` that has been applied to this cursor into account when
getting the count
.. note:: The `with_limit_and_skip` parameter requires server
version **>= 1.1.4-**
.. versionadded:: 1.1.1
The `with_limit_and_skip` parameter.
:meth:`~pymongo.cursor.Cursor.__len__` was deprecated in favor of
calling :meth:`count` with `with_limit_and_skip` set to ``True``.
"""
command = SON([("count", self.__collection.name),
("query", self.__spec), ("fields", self.__fields)])
if with_limit_and_skip:
if self.__limit:
command["limit"] = self.__limit
if self.__skip:
command["skip"] = self.__skip
response = self.__collection.database.command(
command, allowable_errors=["ns missing"])
if response.get("errmsg", "") == "ns missing":
return 0
return int(response["n"])
def distinct(self, key):
"""Get a list of distinct values for `key` among all documents
in the result set of this query.
Raises :class:`TypeError` if `key` is not an instance of
:class:`basestring`.
:Parameters:
- `key`: name of key for which we want to get the distinct values
.. note:: Requires server version **>= 1.1.3+**
.. seealso:: :meth:`pymongo.collection.Collection.distinct`
.. versionadded:: 1.2
"""
if not isinstance(key, basestring):
raise TypeError("key must be an instance of basestring")
command = SON([("distinct", self.__collection.name), ("key", key)])
if self.__spec:
command["query"] = self.__spec
return self.__collection.database.command(command)["values"]
def count(self):
if self.retrieved:
raise ValueError("can't count an already started cursor")
db, col = self.col.split('.', 1)
command = SON([('count', col), ('query', self.spec)])
result = self.client._command(db, command)
return int(result.get('n', 0))
def import_docs(user='', pw='', host='localhost', db_name='capublic'):
if user and pw:
conn_str = 'mysql://%s:%s@' % (user, pw)
else:
conn_str = 'mysql://'
conn_str = '%s%s/%s?charset=utf8&unix_socket=/tmp/mysql.sock' % (
conn_str, host, db_name)
engine = create_engine(conn_str)
Session = sessionmaker(bind=engine)
session = Session()
fs = gridfs.GridFS(db, collection="documents")
for version in session.query(CABillVersion).filter(
CABillVersion.bill_xml != None):
if fs.exists({"metadata": {"ca_version_id": version.bill_version_id}}):
continue
query = SON([('_id', 'ca')])
update = SON([('$inc', SON([('seq', 1)]))])
seq = db.command(
SON([('findandmodify', 'doc_ids'), ('query', query),
('update', update), ('new', True),
('upsert', True)]))['value']['seq']
doc_id = "CAD%08d" % seq
print "Saving: %s" % doc_id
fs.put(version.bill_xml,
_id=doc_id,
content_type='text/xml',
metadata={"ca_version_id": version.bill_version_id})
bill = db.bills.find_one({'versions.name': version.bill_version_id})
if not bill:
print "Couldn't find bill for %s" % version.bill_version_id
continue
for v in bill['versions']:
if v['name'] == version.bill_version_id:
v['url'] = ("http://openstates.sunlightlabs.com/api/"
"documents/%s/" % doc_id)
break
db.bills.save(bill, safe=True)
def sm_object_hook(obj):
if "$pyhint" in obj:
temp = SON()
for pair in obj['$pyhint']:
temp[pair['key']] = pair['value']
return temp
else:
return json_util.object_hook(obj)
def test_create_index(self):
db = self.db
self.assertRaises(TypeError, db.test.create_index, 5)
self.assertRaises(TypeError, db.test.create_index, {"hello": 1})
self.assertRaises(ValueError, db.test.create_index, [])
db.test.drop_indexes()
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 1)
db.test.create_index("hello")
db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)])
count = 0
for _ in db.system.indexes.find({"ns": u"pymongo_test.test"}):
count += 1
self.assertEqual(count, 3)
db.test.drop_indexes()
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 1)
db.test.create_index("hello")
self.assert_(
SON([(u"name",
u"hello_1"), (u"unique", False), (
u"ns",
u"pymongo_test.test"), (u"key", SON([(u"hello", 1)]))])
in list(db.system.indexes.find({"ns": u"pymongo_test.test"})))
db.test.drop_indexes()
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 1)
db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)])
self.assert_(
SON([(u"name",
u"hello_-1_world_1"), (u"unique",
False), (u"ns", u"pymongo_test.test"),
(u"key", SON([(u"hello", -1), (u"world", 1)]))]) in list(
db.system.indexes.find({"ns": u"pymongo_test.test"})))
def post(self):
db = self.get_argument('db')
coll = self.get_argument('coll')
cmd = SON(data={'validate': coll})
result = self.c[db].command(cmd, check=False)
result['result'] = '<pre>' + result[
'result'] + '</pre>' # format the string
self.respond_back_result(result)
def transform_value(value):
if isinstance(value, DBRef):
return self.__database.dereference(value)
elif isinstance(value, list):
return [transform_value(v) for v in value]
elif isinstance(value, dict):
return transform_dict(SON(value))
return value
def transform_incoming(self, son, collection):
"""Move _id to the front if it's there.
"""
if not "_id" in son:
return son
transformed = SON({"_id": son["_id"]})
transformed.update(son)
return transformed
def transform_value(value):
if isinstance(value, dict):
if "_id" in value and "_ns" in value:
return DBRef(value["_ns"], transform_value(value["_id"]))
else:
return transform_dict(SON(value))
elif isinstance(value, list):
return [transform_value(v) for v in value]
return value
def as_doc(self):
"""Get the SON document representation of this DBRef.
Generally not needed by application developers
"""
doc = SON([("$ref", self.collection), ("$id", self.id)])
if self.database is not None:
doc["$db"] = self.database
return doc
def test_id_ordering(self):
db = self.connection.pymongo_test
db.test.remove({})
db.test.insert({"hello": "world", "_id": 5})
db.test.insert(SON([("hello", "world"), ("_id", 5)]))
for x in db.test.find():
for (k, v) in x.items():
self.assertEqual(k, "_id")
break
def transform_outgoing(self, son, collection):
"""Manipulate an outgoing SON object.
:Parameters:
- `son`: the SON object being retrieved from the database
- `collection`: the collection this object was stored in
"""
if self.will_copy():
return SON(son)
return son
def test_drop_index(self):
db = self.db
db.test.drop_indexes()
db.test.create_index("hello")
name = db.test.create_index("goodbye")
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 3)
self.assertEqual(name, "goodbye_1")
db.test.drop_index(name)
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 2)
self.assert_(
SON([(u"name",
u"hello_1"), (u"unique", False), (
u"ns",
u"pymongo_test.test"), (u"key", SON([(u"hello", 1)]))])
in list(db.system.indexes.find({"ns": u"pymongo_test.test"})))
db.test.drop_indexes()
db.test.create_index("hello")
name = db.test.create_index("goodbye")
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 3)
self.assertEqual(name, "goodbye_1")
db.test.drop_index([("goodbye", ASCENDING)])
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 2)
self.assert_(
SON([(u"name",
u"hello_1"), (u"unique", False), (
u"ns",
u"pymongo_test.test"), (u"key", SON([(u"hello", 1)]))])
in list(db.system.indexes.find({"ns": u"pymongo_test.test"})))
