def find_one(self, spec_or_object_id=None, fields=None, _sock=None, _must_use_master=False, _is_command=False): """Get a single object from the database. Raises TypeError if the argument is of an improper type. Returns a single SON object, or None if no result is found. :Parameters: - `spec_or_object_id` (optional): a SON object specifying elements which must be present for a document to be returned OR an instance of ObjectId to be used as the value for an _id query - `fields` (optional): a list of field names that should be included in the returned document ("_id" will always be included) """ spec = spec_or_object_id if spec is None: spec = SON() if isinstance(spec, ObjectId): spec = SON({"_id": spec}) for result in self.find(spec, limit=-1, fields=fields, _sock=_sock, _must_use_master=_must_use_master, _is_command=_is_command): return result return None
def _touch_query(self): if self._query_additions: spec = SON({'query': self.spec}) for k, v in self._query_additions: if k == 'sort': ordering = spec.setdefault('orderby', SON()) ordering.update(v) self.spec = spec
def next_big_id(abbr, letter, collection): query = SON([('_id', abbr)]) update = SON([('$inc', SON([('seq', 1)]))]) seq = db.command( SON([('findandmodify', collection), ('query', query), ('update', update), ('new', True), ('upsert', True)]))['value']['seq'] return "%s%s%08d" % (abbr.upper(), letter, seq)
def get_next_id(self): # Generate a new sequential ID for the vote query = SON([('_id', self.state)]) update = SON([('$inc', SON([('seq', 1)]))]) seq = db.command( SON([('findandmodify', 'vote_ids'), ('query', query), ('update', update), ('new', True), ('upsert', True)]))['value']['seq'] return "%sV%08d" % (self.state.upper(), seq)
def test_from_xml(self): smorgasbord = """ <twonk> <meta/> <doc> <oid name="_id">285a664923b5fcd8ec000000</oid> <int name="the_answer">42</int> <string name="b">foo</string> <boolean name="c">true</boolean> <number name="pi">3.14159265358979</number> <array name="an_array"> <string name="0">x</string> <string name="1">y</string> <string name="2">z</string> <doc name="3"> <string name="subobject">yup</string> </doc> </array> <date name="now">123144452057</date> <ref name="dbref"> <ns>namespace</ns> <oid>ca5c67496c01d896f7010000</oid> </ref> <regex name="regex"> <pattern>foobar</pattern> <options>i</options> </regex> <code name="$where">this is code</code> <null name="mynull"/> </doc> </twonk> """ self.assertEqual( SON.from_xml(smorgasbord), SON([ (u"_id", ObjectId("\x28\x5A\x66\x49\x23\xB5\xFC" "\xD8\xEC\x00\x00\x00")), (u"the_answer", 42), (u"b", u"foo"), (u"c", True), (u"pi", 3.14159265358979), (u"an_array", [u"x", u"y", u"z", SON([(u"subobject", u"yup")])]), (u"now", datetime.datetime(1973, 11, 26, 6, 47, 32, 57000)), (u"dbref", DBRef( "namespace", ObjectId("\xCA\x5C\x67\x49\x6C\x01" "\xD8\x96\xF7\x01\x00\x00"))), (u"regex", re.compile(u"foobar", re.IGNORECASE)), (u"$where", "this is code"), (u"mynull", None), ]))
def test_ordered_dict(self): a = SON() a["hello"] = "world" a["mike"] = "awesome" a["hello_"] = "mike" self.assertEqual(a.items(), [("hello", "world"), ("mike", "awesome"), ("hello_", "mike")]) b = SON({"hello": "world"}) self.assertEqual(b["hello"], "world") self.assertRaises(KeyError, lambda: b["goodbye"])
def getStatuses(): for i in range(4): # 30 secs if i != 0: sleep(10) print_("fetching serverStatus #%s" % (i + 1)) stat = admin.command('serverStatus') statuses.append(stat) status_times.append(str(datetime.datetime.now())) if i == 0: status_diffs.append(None) else: last = statuses[i - 1] time = (stat['uptime'] - last['uptime']) * 1.0 # just to be safe diff = SON() diff['ops_per_sec'] = SON() diff['ops_per_sec']['insert'] = ( stat['opcounters']['insert'] - last['opcounters']['insert']) / time diff['ops_per_sec']['query'] = (stat['opcounters']['query'] - last['opcounters']['query']) / time diff['ops_per_sec']['update'] = ( stat['opcounters']['update'] - last['opcounters']['update']) / time diff['ops_per_sec']['delete'] = ( stat['opcounters']['delete'] - last['opcounters']['delete']) / time diff['ops_per_sec']['getmore'] = ( stat['opcounters']['getmore'] - last['opcounters']['getmore']) / time diff['ops_per_sec']['command'] = ( stat['opcounters']['command'] - last['opcounters']['command']) / time diff['btree_miss_per_sec'] = ( stat['indexCounters']['btree']['misses'] - last['indexCounters']['btree']['misses']) / time try: diff['heap_change'] = stat['extra_info'][ 'heap_usage_bytes'] - last['extra_info']['heap_usage_bytes'] diff['pflts_per_sec'] = ( stat['extra_info']['page_faults'] - last['extra_info']['page_faults']) / time except KeyError: pass # these fields are only on linux status_diffs.append(diff)
def _insert_with_id(event): query = SON([('_id', event['state'])]) update = SON([('$inc', SON([('seq', 1)]))]) seq = db.command( SON([('findandmodify', 'event_ids'), ('query', query), ('update', update), ('new', True), ('upsert', True)]))['value']['seq'] id = "%sE%08d" % (event['state'].upper(), seq) logging.info("Saving as %s" % id) event['_id'] = id db.events.save(event, safe=True) return id
def put_document(doc, content_type, metadata): # Generate a new sequential ID for the document query = SON([('_id', metadata['bill']['state'])]) update = SON([('$inc', SON([('seq', 1)]))]) seq = db.command( SON([('findandmodify', 'doc_ids'), ('query', query), ('update', update), ('new', True), ('upsert', True)]))['value']['seq'] id = "%sD%08d" % (metadata['bill']['state'].upper(), seq) logging.info("Saving as %s" % id) fs.put(doc, _id=id, content_type=content_type, metadata=metadata) return id
def rename(self, new_name): """Rename this collection. If operating in auth mode, client must be authorized as an admin to perform this operation. Raises :class:`TypeError` if `new_name` is not an instance of :class:`basestring`. Raises :class:`~pymongo.errors.InvalidName` if `new_name` is not a valid collection name. :Parameters: - `new_name`: new name for this collection """ if not isinstance(new_name, basestring): raise TypeError("new_name must be an instance of basestring") if not new_name or ".." in new_name: raise InvalidName("collection names cannot be empty") if "$" in new_name: raise InvalidName("collection names must not contain '$'") if new_name[0] == "." or new_name[-1] == ".": raise InvalidName("collecion names must not start or end with '.'") rename_command = SON([("renameCollection", self.__full_name), ("to", "%s.%s" % (self.__database.name, new_name))]) self.__database.connection.admin.command(rename_command)
def drop_index(self, index_or_name): """Drops the specified index on this collection. Can be used on non-existant collections or collections with no indexes. Raises OperationFailure on an error. `index_or_name` can be either an index name (as returned by `create_index`), or an index specifier (as passed to `create_index`). An index specifier should be a list of (key, direction) pairs. Raises TypeError if index is not an instance of (str, unicode, list). :Parameters: - `index_or_name`: index (or name of index) to drop """ name = index_or_name if isinstance(index_or_name, list): name = self._gen_index_name(index_or_name) if not isinstance(name, basestring): raise TypeError("index_or_name must be an index name or list") self.__database.connection._purge_index(self.__database.name, self.__name, name) self.__database.command( SON([("deleteIndexes", self.__name), ("index", name)]), ["ns not found"])
def test_save_find_one(self): db = Database(self.connection, "pymongo_test") db.test.remove({}) a_doc = SON({"hello": u"world"}) a_key = db.test.save(a_doc) self.assert_(isinstance(a_doc["_id"], ObjectId)) self.assertEqual(a_doc["_id"], a_key) self.assertEqual(a_doc, db.test.find_one({"_id": a_doc["_id"]})) self.assertEqual(a_doc, db.test.find_one(a_key)) self.assertEqual(None, db.test.find_one(ObjectId())) self.assertEqual(a_doc, db.test.find_one({"hello": u"world"})) self.assertEqual(None, db.test.find_one({"hello": u"test"})) b = db.test.find_one() b["hello"] = u"mike" db.test.save(b) self.assertNotEqual(a_doc, db.test.find_one(a_key)) self.assertEqual(b, db.test.find_one(a_key)) self.assertEqual(b, db.test.find_one()) count = 0 for _ in db.test.find(): count += 1 self.assertEqual(count, 1)
def check_for_pos_dupes_via_geoloc(attrs, collection_name=None): if attrs['geometry_type'] == "Point": ll = { '$near': [ float(attrs['geometry_centroid'][0]), float(attrs['geometry_centroid'][1]) ] } cc = ClassifierCategories.objects.get( slug=attrs['classifiers']['category']) md = {'$maxDistance': int(cc.duplicate_distance_tolerance)} q = SON(ll) q.update(md) gq = { 'geometry_centroid': q, 'classifiers.category': attrs['classifiers']['category'] } x = query_mongo_db(gq, collection_name=collection_name) if x.has_key('features'): if len(x['features']) > 0: attrs['possible_duplicate'] = True return attrs
def _index_document(index_list): """Helper to generate an index specifying document. Takes a list of (key, direction) pairs. """ if isinstance(index_list, dict): raise TypeError("passing a dict to sort/create_index/hint is not " "allowed - use a list of tuples instead. did you " "mean %r?" % list(index_list.iteritems())) elif not isinstance(index_list, list): raise TypeError("must use a list of (key, direction) pairs, " "not: %r" % index_list) if not len(index_list): raise ValueError("key_or_list must not be the empty list") index = SON() for (key, value) in index_list: if not isinstance(key, basestring): raise TypeError("first item in each key pair must be a string") if not isinstance(value, int): raise TypeError( "second item in each key pair must be ASCENDING or " "DESCENDING") index[key] = value return index
def sort(self, name, direction): if self.retrieved: raise ValueError("can't sort an already started cursor") key = SON() key[name] = direction self._query_additions.append(('sort', key)) return self
def test_from_then_to_dict(self): def helper(dict): self.assertEqual(dict, (BSON.from_dict(dict)).to_dict()) helper({}) helper({"test": u"hello"}) self.assert_( isinstance( BSON.from_dict({ "hello": "world" }).to_dict()["hello"], types.UnicodeType)) helper({"mike": -10120}) helper({"long": long(10)}) helper({"really big long": 2147483648}) helper({u"hello": 0.0013109}) helper({"something": True}) helper({"false": False}) helper({"an array": [1, True, 3.8, u"world"]}) helper({"an object": {"test": u"something"}}) helper({"a binary": Binary("test", 100)}) helper({"a binary": Binary("test", 128)}) helper({"a binary": Binary("test", 254)}) helper({"another binary": Binary("test")}) helper(SON([(u'test dst', datetime.datetime(1993, 4, 4, 2))])) helper({"big float": float(10000000000)}) def from_then_to_dict(dict): return dict == (BSON.from_dict(dict)).to_dict() qcheck.check_unittest(self, from_then_to_dict, qcheck.gen_mongo_dict(3))
def count(self, with_limit_and_skip=False): """Get the size of the results set for this query. Returns the number of documents in the results set for this query. Does not take :meth:`limit` and :meth:`skip` into account by default - set `with_limit_and_skip` to ``True`` if that is the desired behavior. Raises :class:`~pymongo.errors.OperationFailure` on a database error. :Parameters: - `with_limit_and_skip` (optional): take any :meth:`limit` or :meth:`skip` that has been applied to this cursor into account when getting the count .. note:: The `with_limit_and_skip` parameter requires server version **>= 1.1.4-** .. versionadded:: 1.1.1 The `with_limit_and_skip` parameter. :meth:`~pymongo.cursor.Cursor.__len__` was deprecated in favor of calling :meth:`count` with `with_limit_and_skip` set to ``True``. """ command = SON([("count", self.__collection.name), ("query", self.__spec), ("fields", self.__fields)]) if with_limit_and_skip: if self.__limit: command["limit"] = self.__limit if self.__skip: command["skip"] = self.__skip response = self.__collection.database.command( command, allowable_errors=["ns missing"]) if response.get("errmsg", "") == "ns missing": return 0 return int(response["n"])
def distinct(self, key): """Get a list of distinct values for `key` among all documents in the result set of this query. Raises :class:`TypeError` if `key` is not an instance of :class:`basestring`. :Parameters: - `key`: name of key for which we want to get the distinct values .. note:: Requires server version **>= 1.1.3+** .. seealso:: :meth:`pymongo.collection.Collection.distinct` .. versionadded:: 1.2 """ if not isinstance(key, basestring): raise TypeError("key must be an instance of basestring") command = SON([("distinct", self.__collection.name), ("key", key)]) if self.__spec: command["query"] = self.__spec return self.__collection.database.command(command)["values"]
def count(self): if self.retrieved: raise ValueError("can't count an already started cursor") db, col = self.col.split('.', 1) command = SON([('count', col), ('query', self.spec)]) result = self.client._command(db, command) return int(result.get('n', 0))
def import_docs(user='', pw='', host='localhost', db_name='capublic'): if user and pw: conn_str = 'mysql://%s:%s@' % (user, pw) else: conn_str = 'mysql://' conn_str = '%s%s/%s?charset=utf8&unix_socket=/tmp/mysql.sock' % ( conn_str, host, db_name) engine = create_engine(conn_str) Session = sessionmaker(bind=engine) session = Session() fs = gridfs.GridFS(db, collection="documents") for version in session.query(CABillVersion).filter( CABillVersion.bill_xml != None): if fs.exists({"metadata": {"ca_version_id": version.bill_version_id}}): continue query = SON([('_id', 'ca')]) update = SON([('$inc', SON([('seq', 1)]))]) seq = db.command( SON([('findandmodify', 'doc_ids'), ('query', query), ('update', update), ('new', True), ('upsert', True)]))['value']['seq'] doc_id = "CAD%08d" % seq print "Saving: %s" % doc_id fs.put(version.bill_xml, _id=doc_id, content_type='text/xml', metadata={"ca_version_id": version.bill_version_id}) bill = db.bills.find_one({'versions.name': version.bill_version_id}) if not bill: print "Couldn't find bill for %s" % version.bill_version_id continue for v in bill['versions']: if v['name'] == version.bill_version_id: v['url'] = ("http://openstates.sunlightlabs.com/api/" "documents/%s/" % doc_id) break db.bills.save(bill, safe=True)
def sm_object_hook(obj): if "$pyhint" in obj: temp = SON() for pair in obj['$pyhint']: temp[pair['key']] = pair['value'] return temp else: return json_util.object_hook(obj)
def test_create_index(self): db = self.db self.assertRaises(TypeError, db.test.create_index, 5) self.assertRaises(TypeError, db.test.create_index, {"hello": 1}) self.assertRaises(ValueError, db.test.create_index, []) db.test.drop_indexes() self.assertEqual( db.system.indexes.find({ "ns": u"pymongo_test.test" }).count(), 1) db.test.create_index("hello") db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)]) count = 0 for _ in db.system.indexes.find({"ns": u"pymongo_test.test"}): count += 1 self.assertEqual(count, 3) db.test.drop_indexes() self.assertEqual( db.system.indexes.find({ "ns": u"pymongo_test.test" }).count(), 1) db.test.create_index("hello") self.assert_( SON([(u"name", u"hello_1"), (u"unique", False), ( u"ns", u"pymongo_test.test"), (u"key", SON([(u"hello", 1)]))]) in list(db.system.indexes.find({"ns": u"pymongo_test.test"}))) db.test.drop_indexes() self.assertEqual( db.system.indexes.find({ "ns": u"pymongo_test.test" }).count(), 1) db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)]) self.assert_( SON([(u"name", u"hello_-1_world_1"), (u"unique", False), (u"ns", u"pymongo_test.test"), (u"key", SON([(u"hello", -1), (u"world", 1)]))]) in list( db.system.indexes.find({"ns": u"pymongo_test.test"})))
def post(self): db = self.get_argument('db') coll = self.get_argument('coll') cmd = SON(data={'validate': coll}) result = self.c[db].command(cmd, check=False) result['result'] = '<pre>' + result[ 'result'] + '</pre>' # format the string self.respond_back_result(result)
def transform_value(value): if isinstance(value, DBRef): return self.__database.dereference(value) elif isinstance(value, list): return [transform_value(v) for v in value] elif isinstance(value, dict): return transform_dict(SON(value)) return value
def transform_incoming(self, son, collection): """Move _id to the front if it's there. """ if not "_id" in son: return son transformed = SON({"_id": son["_id"]}) transformed.update(son) return transformed
def transform_value(value): if isinstance(value, dict): if "_id" in value and "_ns" in value: return DBRef(value["_ns"], transform_value(value["_id"])) else: return transform_dict(SON(value)) elif isinstance(value, list): return [transform_value(v) for v in value] return value
def as_doc(self): """Get the SON document representation of this DBRef. Generally not needed by application developers """ doc = SON([("$ref", self.collection), ("$id", self.id)]) if self.database is not None: doc["$db"] = self.database return doc
def test_id_ordering(self): db = self.connection.pymongo_test db.test.remove({}) db.test.insert({"hello": "world", "_id": 5}) db.test.insert(SON([("hello", "world"), ("_id", 5)])) for x in db.test.find(): for (k, v) in x.items(): self.assertEqual(k, "_id") break
def transform_outgoing(self, son, collection): """Manipulate an outgoing SON object. :Parameters: - `son`: the SON object being retrieved from the database - `collection`: the collection this object was stored in """ if self.will_copy(): return SON(son) return son
def test_drop_index(self): db = self.db db.test.drop_indexes() db.test.create_index("hello") name = db.test.create_index("goodbye") self.assertEqual( db.system.indexes.find({ "ns": u"pymongo_test.test" }).count(), 3) self.assertEqual(name, "goodbye_1") db.test.drop_index(name) self.assertEqual( db.system.indexes.find({ "ns": u"pymongo_test.test" }).count(), 2) self.assert_( SON([(u"name", u"hello_1"), (u"unique", False), ( u"ns", u"pymongo_test.test"), (u"key", SON([(u"hello", 1)]))]) in list(db.system.indexes.find({"ns": u"pymongo_test.test"}))) db.test.drop_indexes() db.test.create_index("hello") name = db.test.create_index("goodbye") self.assertEqual( db.system.indexes.find({ "ns": u"pymongo_test.test" }).count(), 3) self.assertEqual(name, "goodbye_1") db.test.drop_index([("goodbye", ASCENDING)]) self.assertEqual( db.system.indexes.find({ "ns": u"pymongo_test.test" }).count(), 2) self.assert_( SON([(u"name", u"hello_1"), (u"unique", False), ( u"ns", u"pymongo_test.test"), (u"key", SON([(u"hello", 1)]))]) in list(db.system.indexes.find({"ns": u"pymongo_test.test"})))