def map_reduce(self, map, reduce, full_response=False, **kwargs):
"""Perform a map/reduce operation on this collection.
If `full_response` is ``False`` (default) returns a
:class:`~pymongo.collection.Collection` instance containing
the results of the operation. Otherwise, returns the full
response from the server to the `map reduce command`_.
:Parameters:
- `map`: map function (as a JavaScript string)
- `reduce`: reduce function (as a JavaScript string)
- `full_response` (optional): if ``True``, return full response to
this command - otherwise just return the result collection
- `**kwargs` (optional): additional arguments to the
`map reduce command`_ may be passed as keyword arguments to this
helper method, e.g.::
>>> db.test.map_reduce(map, reduce, limit=2)
.. note:: Requires server version **>= 1.1.1**
.. seealso:: :doc:`/examples/map_reduce`
.. versionadded:: 1.2
.. _map reduce command: http://www.mongodb.org/display/DOCS/MapReduce
"""
command = SON([("mapreduce", self.__name),
("map", map), ("reduce", reduce)])
command.update(**kwargs)
response = self.__database.command(command)
if full_response:
return response
return self.__database[response["result"]]
def put_documentesribyid(id):
data = request.body.readline()
result = json.loads(data)
print data
entity = db['documents'].find_one({'_id':id})
if not entity:
entity = result
tmp = {}
tmp["features"] = [entity]
tmp["geometryType"] = entity["geometryType"]
geojs = esri_to_geo(tmp)
geojs["_id"] = entity["_id"]
near_dict = {"$near": geojs["geometry"]["coordinates"]}
max_dict = max_dict = {"$maxDistance": 0.001}
q = SON(near_dict)
q.update(max_dict)
gq = {"geometry.coordinates": q}
check = db["documents"].find(gq)
print check
if check:
abort(400, {"success": False, "error": "Already exist"})
else:
geojs = entity
geojs["properties"]["votes"] = result["attributes"]["votes"]
try:
db['documents'].save(geojs)
except ValidationError as ve:
abort(400, str(ve))
return result
def check_for_pos_dupes_via_geoloc(attrs, collection_name=None):
if attrs['geometry_type'] == "Point":
ll = {
'$near': [
float(attrs['geometry_centroid'][0]),
float(attrs['geometry_centroid'][1])
]
}
cc = ClassifierCategories.objects.get(
slug=attrs['classifiers']['category'])
md = {'$maxDistance': int(cc.duplicate_distance_tolerance)}
q = SON(ll)
q.update(md)
gq = {
'geometry_centroid': q,
'classifiers.category': attrs['classifiers']['category']
}
x = query_mongo_db(gq, collection_name=collection_name)
if x.has_key('features'):
if len(x['features']) > 0:
attrs['possible_duplicate'] = True
return attrs
def find_one(self,
spec_or_object_id=None,
fields=None,
_sock=None,
_must_use_master=False,
_is_command=False):
"""Get a single object from the database.
Raises TypeError if the argument is of an improper type. Returns a
single SON object, or None if no result is found.
:Parameters:
- `spec_or_object_id` (optional): a SON object specifying elements
which must be present for a document to be returned OR an instance
of ObjectId to be used as the value for an _id query
- `fields` (optional): a list of field names that should be included
in the returned document ("_id" will always be included)
"""
spec = spec_or_object_id
if spec is None:
spec = SON()
if isinstance(spec, ObjectId):
spec = SON({"_id": spec})
for result in self.find(spec,
limit=-1,
fields=fields,
_sock=_sock,
_must_use_master=_must_use_master,
_is_command=_is_command):
return result
return None
def transform_incoming(self, son, collection):
"""Move _id to the front if it's there.
"""
if not "_id" in son:
return son
transformed = SON({"_id": son["_id"]})
transformed.update(son)
return transformed
def _touch_query(self):
if self._query_additions:
spec = SON({'$query': self.spec or {}})
for k, v in self._query_additions:
if k == 'sort':
ordering = spec.setdefault('$orderby', SON())
ordering.update(v)
self.spec = spec
def next_big_id(abbr, letter, collection):
query = SON([('_id', abbr)])
update = SON([('$inc', SON([('seq', 1)]))])
seq = db.command(
SON([('findandmodify', collection),
('query', query), ('update', update), ('new', True),
('upsert', True)]))['value']['seq']
return "%s%s%08d" % (abbr.upper(), letter, seq)
def _touch_query(self):
if self._query_additions:
spec = SON({'query': self.spec})
for k, v in self._query_additions:
if k == 'sort':
ordering = spec.setdefault('orderby', SON())
ordering.update(v)
self.spec = spec
def transform_incoming(self, son, collection):
"""Move _id to the front if it's there.
"""
if not "_id" in son:
return son
transformed = SON({"_id": son["_id"]})
transformed.update(son)
return transformed
def get_next_id(self):
# Generate a new sequential ID for the vote
query = SON([('_id', self.state)])
update = SON([('$inc', SON([('seq', 1)]))])
seq = db.command(
SON([('findandmodify', 'vote_ids'), ('query', query),
('update', update), ('new', True),
('upsert', True)]))['value']['seq']
return "%sV%08d" % (self.state.upper(), seq)
def test_ordered_dict(self):
a = SON()
a["hello"] = "world"
a["mike"] = "awesome"
a["hello_"] = "mike"
self.assertEqual(a.items(), [("hello", "world"), ("mike", "awesome"), ("hello_", "mike")])
b = SON({"hello": "world"})
self.assertEqual(b["hello"], "world")
self.assertRaises(KeyError, lambda: b["goodbye"])
def test_from_xml(self):
smorgasbord = """
<twonk>
<meta/>
<doc>
<oid name="_id">285a664923b5fcd8ec000000</oid>
<int name="the_answer">42</int>
<string name="b">foo</string>
<boolean name="c">true</boolean>
<number name="pi">3.14159265358979</number>
<array name="an_array">
<string name="0">x</string>
<string name="1">y</string>
<string name="2">z</string>
<doc name="3">
<string name="subobject">yup</string>
</doc>
</array>
<date name="now">123144452057</date>
<ref name="dbref">
<ns>namespace</ns>
<oid>ca5c67496c01d896f7010000</oid>
</ref>
<regex name="regex">
<pattern>foobar</pattern>
<options>i</options>
</regex>
<code name="$where">this is code</code>
<null name="mynull"/>
</doc>
</twonk>
"""
self.assertEqual(
SON.from_xml(smorgasbord),
SON([
(u"_id",
ObjectId("\x28\x5A\x66\x49\x23\xB5\xFC"
"\xD8\xEC\x00\x00\x00")),
(u"the_answer", 42),
(u"b", u"foo"),
(u"c", True),
(u"pi", 3.14159265358979),
(u"an_array",
[u"x", u"y", u"z",
SON([(u"subobject", u"yup")])]),
(u"now", datetime.datetime(1973, 11, 26, 6, 47, 32, 57000)),
(u"dbref",
DBRef(
"namespace",
ObjectId("\xCA\x5C\x67\x49\x6C\x01"
"\xD8\x96\xF7\x01\x00\x00"))),
(u"regex", re.compile(u"foobar", re.IGNORECASE)),
(u"$where", "this is code"),
(u"mynull", None),
]))
def test_ordered_dict(self):
a = SON()
a["hello"] = "world"
a["mike"] = "awesome"
a["hello_"] = "mike"
self.assertEqual(a.items(), [("hello", "world"), ("mike", "awesome"),
("hello_", "mike")])
b = SON({"hello": "world"})
self.assertEqual(b["hello"], "world")
self.assertRaises(KeyError, lambda: b["goodbye"])
def as_doc(self):
"""Get the SON document representation of this DBRef.
Generally not needed by application developers
"""
doc = SON([("$ref", self.collection),
("$id", self.id)])
if self.database is not None:
doc["$db"] = self.database
doc.update(self.__kwargs)
return doc
def getDuplicateCandidates(self, entity):
if not 'lat' in entity or not 'lng' in entity:
raise Fail('invalid entity')
# TODO: verify lat / lng versus lng / lat
q = SON({"$near" : [entity.lng, entity.lat]})
q.update({"$maxDistance" : self.distance })
docs = self._placesDB._collection.find({"coordinates" : q}, output=list)
entities = self._gen_entities(docs)
return entities
def __create(self, options):
"""Sends a create command with the given options.
"""
# Send size as a float, not an int/long. BSON can only handle 32-bit
# ints which conflicts w/ max collection size of 10000000000.
if "size" in options:
options["size"] = float(options["size"])
command = SON({"create": self.__name})
command.update(options)
self.__database.command(command)
def __create(self, options):
"""Sends a create command with the given options.
"""
# Send size as a float, not an int/long. BSON can only handle 32-bit
# ints which conflicts w/ max collection size of 10000000000.
if "size" in options:
options["size"] = float(options["size"])
command = SON({"create": self.__name})
command.update(options)
self.__database.command(command)
def getStatuses():
for i in range(4): # 30 secs
if i != 0:
sleep(10)
print_("fetching serverStatus #%s" % (i + 1))
stat = admin.command('serverStatus')
statuses.append(stat)
status_times.append(str(datetime.datetime.now()))
if i == 0:
status_diffs.append(None)
else:
last = statuses[i - 1]
time = (stat['uptime'] - last['uptime']) * 1.0 # just to be safe
diff = SON()
diff['ops_per_sec'] = SON()
diff['ops_per_sec']['insert'] = (
stat['opcounters']['insert'] -
last['opcounters']['insert']) / time
diff['ops_per_sec']['query'] = (stat['opcounters']['query'] -
last['opcounters']['query']) / time
diff['ops_per_sec']['update'] = (
stat['opcounters']['update'] -
last['opcounters']['update']) / time
diff['ops_per_sec']['delete'] = (
stat['opcounters']['delete'] -
last['opcounters']['delete']) / time
diff['ops_per_sec']['getmore'] = (
stat['opcounters']['getmore'] -
last['opcounters']['getmore']) / time
diff['ops_per_sec']['command'] = (
stat['opcounters']['command'] -
last['opcounters']['command']) / time
diff['btree_miss_per_sec'] = (
stat['indexCounters']['btree']['misses'] -
last['indexCounters']['btree']['misses']) / time
try:
diff['heap_change'] = stat['extra_info'][
'heap_usage_bytes'] - last['extra_info']['heap_usage_bytes']
diff['pflts_per_sec'] = (
stat['extra_info']['page_faults'] -
last['extra_info']['page_faults']) / time
except KeyError:
pass # these fields are only on linux
status_diffs.append(diff)
def put_document(doc, content_type, metadata):
# Generate a new sequential ID for the document
query = SON([('_id', metadata['bill']['state'])])
update = SON([('$inc', SON([('seq', 1)]))])
seq = db.command(
SON([('findandmodify', 'doc_ids'),
('query', query), ('update', update), ('new', True),
('upsert', True)]))['value']['seq']
id = "%sD%08d" % (metadata['bill']['state'].upper(), seq)
logging.info("Saving as %s" % id)
fs.put(doc, _id=id, content_type=content_type, metadata=metadata)
return id
def _insert_with_id(event):
query = SON([('_id', event['state'])])
update = SON([('$inc', SON([('seq', 1)]))])
seq = db.command(
SON([('findandmodify', 'event_ids'), ('query', query),
('update', update), ('new', True),
('upsert', True)]))['value']['seq']
id = "%sE%08d" % (event['state'].upper(), seq)
logging.info("Saving as %s" % id)
event['_id'] = id
db.events.save(event, safe=True)
return id
def test_data_files(self):
# TODO don't hardcode this, actually clone the repo
data_files = "../mongo-qa/modules/bson_tests/tests/*/*.xson"
generate = True
for file_name in glob.glob(data_files):
f = open(file_name, "r")
xml = f.read()
f.close()
try:
doc = SON.from_xml(xml)
bson = BSON.from_dict(doc)
except UnsupportedTag:
print "skipped file %s: %s" % (file_name, sys.exc_info()[1])
continue
try:
f = open(file_name.replace(".xson", ".bson"), "rb")
expected = f.read()
f.close()
self.assertEqual(bson, expected, file_name)
self.assertEqual(doc, bson.to_dict(), file_name)
except IOError:
if generate:
print "generating .bson for %s" % file_name
f = open(file_name.replace(".xson", ".bson"), "w")
f.write(bson)
f.close()
def sort(self, name, direction):
if self.retrieved:
raise ValueError("can't sort an already started cursor")
key = SON()
key[name] = direction
self._query_additions.append(('sort', key))
return self
def _index_document(index_list):
"""Helper to generate an index specifying document.
Takes a list of (key, direction) pairs.
"""
if isinstance(index_list, dict):
raise TypeError("passing a dict to sort/create_index/hint is not "
"allowed - use a list of tuples instead. did you "
"mean %r?" % list(index_list.iteritems()))
elif not isinstance(index_list, list):
raise TypeError("must use a list of (key, direction) pairs, "
"not: %r" % index_list)
if not len(index_list):
raise ValueError("key_or_list must not be the empty list")
index = SON()
for (key, value) in index_list:
if not isinstance(key, basestring):
raise TypeError("first item in each key pair must be a string")
if not isinstance(value, int):
raise TypeError(
"second item in each key pair must be ASCENDING or "
"DESCENDING")
index[key] = value
return index
def test_save_find_one(self):
db = Database(self.connection, "pymongo_test")
db.test.remove({})
a_doc = SON({"hello": u"world"})
a_key = db.test.save(a_doc)
self.assert_(isinstance(a_doc["_id"], ObjectId))
self.assertEqual(a_doc["_id"], a_key)
self.assertEqual(a_doc, db.test.find_one({"_id": a_doc["_id"]}))
self.assertEqual(a_doc, db.test.find_one(a_key))
self.assertEqual(None, db.test.find_one(ObjectId()))
self.assertEqual(a_doc, db.test.find_one({"hello": u"world"}))
self.assertEqual(None, db.test.find_one({"hello": u"test"}))
b = db.test.find_one()
b["hello"] = u"mike"
db.test.save(b)
self.assertNotEqual(a_doc, db.test.find_one(a_key))
self.assertEqual(b, db.test.find_one(a_key))
self.assertEqual(b, db.test.find_one())
count = 0
for _ in db.test.find():
count += 1
self.assertEqual(count, 1)
def get_features_near_point(request, lat, lon, max_distance=None, limit=None):
"""
Return a geographic features located neat these coordinates.
"""
ll = {'$near':[float(lat), float(lon)]}
if max_distance:
md={'$maxDistance': max_distance}
q=SON(ll)
q.update(md)
return get_document_by(request, limit=limit, search_in=('url', 'get'),
geometry_coordinates=q,
)
else:
return get_document_by(request, limit=limit, search_in=('url', 'get'),
geometry_coordinates=ll,
)
def count(self, with_limit_and_skip=False):
"""Get the size of the results set for this query.
Returns the number of documents in the results set for this query. Does
not take :meth:`limit` and :meth:`skip` into account by default - set
`with_limit_and_skip` to ``True`` if that is the desired behavior.
Raises :class:`~pymongo.errors.OperationFailure` on a database error.
:Parameters:
- `with_limit_and_skip` (optional): take any :meth:`limit` or
:meth:`skip` that has been applied to this cursor into account when
getting the count
.. note:: The `with_limit_and_skip` parameter requires server
version **>= 1.1.4-**
.. versionadded:: 1.1.1
The `with_limit_and_skip` parameter.
:meth:`~pymongo.cursor.Cursor.__len__` was deprecated in favor of
calling :meth:`count` with `with_limit_and_skip` set to ``True``.
"""
command = SON([("count", self.__collection.name),
("query", self.__spec), ("fields", self.__fields)])
if with_limit_and_skip:
if self.__limit:
command["limit"] = self.__limit
if self.__skip:
command["skip"] = self.__skip
response = self.__collection.database.command(
command, allowable_errors=["ns missing"])
if response.get("errmsg", "") == "ns missing":
return 0
return int(response["n"])
def distinct(self, key):
"""Get a list of distinct values for `key` among all documents
in the result set of this query.
Raises :class:`TypeError` if `key` is not an instance of
:class:`basestring`.
:Parameters:
- `key`: name of key for which we want to get the distinct values
.. note:: Requires server version **>= 1.1.3+**
.. seealso:: :meth:`pymongo.collection.Collection.distinct`
.. versionadded:: 1.2
"""
if not isinstance(key, basestring):
raise TypeError("key must be an instance of basestring")
command = SON([("distinct", self.__collection.name), ("key", key)])
if self.__spec:
command["query"] = self.__spec
return self.__collection.database.command(command)["values"]
def test_data_files(self):
# TODO don't hardcode this, actually clone the repo
data_files = "../mongo-qa/modules/bson_tests/tests/*/*.xson"
generate = True
for file_name in glob.glob(data_files):
f = open(file_name, "r")
xml = f.read()
f.close()
try:
doc = SON.from_xml(xml)
bson = BSON.from_dict(doc)
except UnsupportedTag:
print "skipped file %s: %s" % (file_name, sys.exc_info()[1])
continue
try:
f = open(file_name.replace(".xson", ".bson"), "rb")
expected = f.read()
f.close()
self.assertEqual(bson, expected, file_name)
self.assertEqual(doc, bson.to_dict(), file_name)
except IOError:
if generate:
print "generating .bson for %s" % file_name
f = open(file_name.replace(".xson", ".bson"), "w")
f.write(bson)
f.close()
def count(self):
if self.retrieved:
raise ValueError("can't count an already started cursor")
db, col = self.col.split('.', 1)
command = SON([('count', col), ('query', self.spec)])
result = self.client._command(db, command)
return int(result.get('n', 0))
def drop_index(self, index_or_name):
"""Drops the specified index on this collection.
Can be used on non-existant collections or collections with no indexes.
Raises OperationFailure on an error. `index_or_name` can be either an
index name (as returned by `create_index`), or an index specifier (as
passed to `create_index`). An index specifier should be a list of (key,
direction) pairs. Raises TypeError if index is not an instance of (str,
unicode, list).
:Parameters:
- `index_or_name`: index (or name of index) to drop
"""
name = index_or_name
if isinstance(index_or_name, list):
name = self._gen_index_name(index_or_name)
if not isinstance(name, basestring):
raise TypeError("index_or_name must be an index name or list")
self.__database.connection._purge_index(self.__database.name,
self.__name, name)
self.__database.command(
SON([("deleteIndexes", self.__name), ("index", name)]),
["ns not found"])
def test_from_then_to_dict(self):
def helper(dict):
self.assertEqual(dict, (BSON.from_dict(dict)).to_dict())
helper({})
helper({"test": u"hello"})
self.assert_(
isinstance(
BSON.from_dict({
"hello": "world"
}).to_dict()["hello"], types.UnicodeType))
helper({"mike": -10120})
helper({"long": long(10)})
helper({"really big long": 2147483648})
helper({u"hello": 0.0013109})
helper({"something": True})
helper({"false": False})
helper({"an array": [1, True, 3.8, u"world"]})
helper({"an object": {"test": u"something"}})
helper({"a binary": Binary("test", 100)})
helper({"a binary": Binary("test", 128)})
helper({"a binary": Binary("test", 254)})
helper({"another binary": Binary("test")})
helper(SON([(u'test dst', datetime.datetime(1993, 4, 4, 2))]))
helper({"big float": float(10000000000)})
def from_then_to_dict(dict):
return dict == (BSON.from_dict(dict)).to_dict()
qcheck.check_unittest(self, from_then_to_dict,
qcheck.gen_mongo_dict(3))
def rename(self, new_name):
"""Rename this collection.
If operating in auth mode, client must be authorized as an
admin to perform this operation. Raises :class:`TypeError` if
`new_name` is not an instance of :class:`basestring`. Raises
:class:`~pymongo.errors.InvalidName` if `new_name` is not a
valid collection name.
:Parameters:
- `new_name`: new name for this collection
"""
if not isinstance(new_name, basestring):
raise TypeError("new_name must be an instance of basestring")
if not new_name or ".." in new_name:
raise InvalidName("collection names cannot be empty")
if "$" in new_name:
raise InvalidName("collection names must not contain '$'")
if new_name[0] == "." or new_name[-1] == ".":
raise InvalidName("collecion names must not start or end with '.'")
rename_command = SON([("renameCollection", self.__full_name),
("to",
"%s.%s" % (self.__database.name, new_name))])
self.__database.connection.admin.command(rename_command)
def import_docs(user='', pw='', host='localhost', db_name='capublic'):
if user and pw:
conn_str = 'mysql://%s:%s@' % (user, pw)
else:
conn_str = 'mysql://'
conn_str = '%s%s/%s?charset=utf8&unix_socket=/tmp/mysql.sock' % (
conn_str, host, db_name)
engine = create_engine(conn_str)
Session = sessionmaker(bind=engine)
session = Session()
fs = gridfs.GridFS(db, collection="documents")
for version in session.query(CABillVersion).filter(
CABillVersion.bill_xml != None):
if fs.exists({"metadata": {"ca_version_id": version.bill_version_id}}):
continue
query = SON([('_id', 'ca')])
update = SON([('$inc', SON([('seq', 1)]))])
seq = db.command(
SON([('findandmodify', 'doc_ids'), ('query', query),
('update', update), ('new', True),
('upsert', True)]))['value']['seq']
doc_id = "CAD%08d" % seq
print "Saving: %s" % doc_id
fs.put(version.bill_xml,
_id=doc_id,
content_type='text/xml',
metadata={"ca_version_id": version.bill_version_id})
bill = db.bills.find_one({'versions.name': version.bill_version_id})
if not bill:
print "Couldn't find bill for %s" % version.bill_version_id
continue
for v in bill['versions']:
if v['name'] == version.bill_version_id:
v['url'] = ("http://openstates.sunlightlabs.com/api/"
"documents/%s/" % doc_id)
break
db.bills.save(bill, safe=True)
def transform_value(value):
if isinstance(value, DBRef):
return self.__database.dereference(value)
elif isinstance(value, list):
return [transform_value(v) for v in value]
elif isinstance(value, dict):
return transform_dict(SON(value))
return value
def sm_object_hook(obj):
if "$pyhint" in obj:
temp = SON()
for pair in obj['$pyhint']:
temp[pair['key']] = pair['value']
return temp
else:
return json_util.object_hook(obj)
def test_create_index(self):
db = self.db
self.assertRaises(TypeError, db.test.create_index, 5)
self.assertRaises(TypeError, db.test.create_index, {"hello": 1})
self.assertRaises(ValueError, db.test.create_index, [])
db.test.drop_indexes()
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 1)
db.test.create_index("hello")
db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)])
count = 0
for _ in db.system.indexes.find({"ns": u"pymongo_test.test"}):
count += 1
self.assertEqual(count, 3)
db.test.drop_indexes()
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 1)
db.test.create_index("hello")
self.assert_(
SON([(u"name",
u"hello_1"), (u"unique", False), (
u"ns",
u"pymongo_test.test"), (u"key", SON([(u"hello", 1)]))])
in list(db.system.indexes.find({"ns": u"pymongo_test.test"})))
db.test.drop_indexes()
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 1)
db.test.create_index([("hello", DESCENDING), ("world", ASCENDING)])
self.assert_(
SON([(u"name",
u"hello_-1_world_1"), (u"unique",
False), (u"ns", u"pymongo_test.test"),
(u"key", SON([(u"hello", -1), (u"world", 1)]))]) in list(
db.system.indexes.find({"ns": u"pymongo_test.test"})))
def post(self):
db = self.get_argument('db')
coll = self.get_argument('coll')
cmd = SON(data={'validate': coll})
result = self.c[db].command(cmd, check=False)
result['result'] = '<pre>' + result[
'result'] + '</pre>' # format the string
self.respond_back_result(result)
def simplify(case): # TODO this is a hack
if isinstance(case, SON) and "$ref" not in case:
simplified = SON(case) # make a copy!
if random.choice([True, False]):
# delete
if not len(simplified.keys()):
return (False, case)
del simplified[random.choice(simplified.keys())]
return (True, simplified)
else:
# simplify a value
if not len(simplified.items()):
return (False, case)
(key, value) = random.choice(simplified.items())
(success, value) = simplify(value)
simplified[key] = value
return (success, success and simplified or case)
if isinstance(case, list):
simplified = list(case)
if random.choice([True, False]):
# delete
if not len(simplified):
return (False, case)
simplified.pop(random.randrange(len(simplified)))
return (True, simplified)
else:
# simplify an item
if not len(simplified):
return (False, case)
index = random.randrange(len(simplified))
(success, value) = simplify(simplified[index])
simplified[index] = value
return (success, success and simplified or case)
return (False, case)
def transform_value(value):
if isinstance(value, dict):
if "_id" in value and "_ns" in value:
return DBRef(value["_ns"], transform_value(value["_id"]))
else:
return transform_dict(SON(value))
elif isinstance(value, list):
return [transform_value(v) for v in value]
return value
def check_for_pos_dupes_via_geoloc(attrs, collection_name=None):
if attrs['geometry_type']=="Point":
ll = {'$near':[float(attrs['geometry_centroid'][0]),
float(attrs['geometry_centroid'][1] )]}
cc = ClassifierCategories.objects.get(slug=attrs['classifiers']['category'])
md={'$maxDistance': int(cc.duplicate_distance_tolerance)}
q=SON(ll)
q.update(md)
gq={'geometry_centroid': q,
'classifiers.category': attrs['classifiers']['category']}
x=query_mongo_db(gq, collection_name=collection_name)
if x.has_key('features'):
if len(x['features'])>0:
attrs['possible_duplicate']=True
return attrs
def as_doc(self):
"""Get the SON document representation of this DBRef.
Generally not needed by application developers
"""
doc = SON([("$ref", self.collection), ("$id", self.id)])
if self.database is not None:
doc["$db"] = self.database
return doc
def test_drop_index(self):
db = self.db
db.test.drop_indexes()
db.test.create_index("hello")
name = db.test.create_index("goodbye")
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 3)
self.assertEqual(name, "goodbye_1")
db.test.drop_index(name)
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 2)
self.assert_(
SON([(u"name",
u"hello_1"), (u"unique", False), (
u"ns",
u"pymongo_test.test"), (u"key", SON([(u"hello", 1)]))])
in list(db.system.indexes.find({"ns": u"pymongo_test.test"})))
db.test.drop_indexes()
db.test.create_index("hello")
name = db.test.create_index("goodbye")
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 3)
self.assertEqual(name, "goodbye_1")
db.test.drop_index([("goodbye", ASCENDING)])
self.assertEqual(
db.system.indexes.find({
"ns": u"pymongo_test.test"
}).count(), 2)
self.assert_(
SON([(u"name",
u"hello_1"), (u"unique", False), (
u"ns",
u"pymongo_test.test"), (u"key", SON([(u"hello", 1)]))])
in list(db.system.indexes.find({"ns": u"pymongo_test.test"})))
def test_from_xml(self):
smorgasbord = """
<twonk>
<meta/>
<doc>
<oid name="_id">285a664923b5fcd8ec000000</oid>
<int name="the_answer">42</int>
<string name="b">foo</string>
<boolean name="c">true</boolean>
<number name="pi">3.14159265358979</number>
<array name="an_array">
<string name="0">x</string>
<string name="1">y</string>
<string name="2">z</string>
<doc name="3">
<string name="subobject">yup</string>
</doc>
</array>
<date name="now">123144452057</date>
<ref name="dbref">
<ns>namespace</ns>
<oid>ca5c67496c01d896f7010000</oid>
</ref>
<regex name="regex">
<pattern>foobar</pattern>
<options>i</options>
</regex>
<code name="$where">this is code</code>
<null name="mynull"/>
</doc>
</twonk>
"""
self.assertEqual(
SON.from_xml(smorgasbord),
SON(
[
(u"_id", ObjectId("\x28\x5A\x66\x49\x23\xB5\xFC" "\xD8\xEC\x00\x00\x00")),
(u"the_answer", 42),
(u"b", u"foo"),
(u"c", True),
(u"pi", 3.14159265358979),
(u"an_array", [u"x", u"y", u"z", SON([(u"subobject", u"yup")])]),
(u"now", datetime.datetime(1973, 11, 26, 6, 47, 32, 57000)),
(u"dbref", DBRef("namespace", ObjectId("\xCA\x5C\x67\x49\x6C\x01" "\xD8\x96\xF7\x01\x00\x00"))),
(u"regex", re.compile(u"foobar", re.IGNORECASE)),
(u"$where", "this is code"),
(u"mynull", None),
]
),
)
def main():
xml_file = sys.argv[1]
out_file = sys.argv[2]
f = open(xml_file, "r")
xml = f.read()
f.close()
f = open(out_file, "w")
doc = SON.from_xml(xml)
bson = BSON.from_dict(doc)
f.write(bson)
f.close()
assert doc == bson.to_dict()
def test_to_dict(self):
a = SON()
b = SON([("blah", SON())])
c = SON([("blah", [SON()])])
d = SON([("blah", {"foo": SON()})])
self.assertEqual({}, a.to_dict())
self.assertEqual({"blah": {}}, b.to_dict())
self.assertEqual({"blah": [{}]}, c.to_dict())
self.assertEqual({"blah": {"foo": {}}}, d.to_dict())
self.assertEqual(dict, a.to_dict().__class__)
self.assertEqual(dict, b.to_dict()["blah"].__class__)
self.assertEqual(dict, c.to_dict()["blah"][0].__class__)
self.assertEqual(dict, d.to_dict()["blah"]["foo"].__class__)
def __last_error(args):
"""Data to send to do a lastError.
"""
cmd = SON([("getlasterror", 1)])
cmd.update(args)
return query(0, "admin.$cmd", 0, -1, cmd)
def toJSON(obj):
return json.dumps(obj, indent=4, default=pymongo.json_util.default)
all_threads = []
def runInThreadNow(func):
t = threading.Thread(target=func)
t.start()
all_threads.append(t)
def cmdName(cmd):
if isinstance(cmd, basestring):
return cmd
else:
return ' '.join(cmd)
cmd_output = SON()
def runOnce(cmd):
print_("running %s"%cmdName(cmd))
output = []
cmd_output[cmdName(cmd)] = output
@runInThreadNow
def runner():
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=isinstance(cmd, basestring))
if proc.wait() != 0:
print_("#### '%s' failed to execute properly. check output for details"%cmdName(cmd))
output.append(proc.stdout.read())
def runForAWhile(cmd, secs=30):
print_("running %s"%cmdName(cmd))
def command(self, command, value=1,
check=True, allowable_errors=[], _sock=None, **kwargs):
"""Issue a MongoDB command.
Send command `command` to the database and return the
response. If `command` is an instance of :class:`basestring`
then the command {`command`: `value`} will be sent. Otherwise,
`command` must be an instance of :class:`dict` and will be
sent as is.
Any additional keyword arguments will be added to the final
command document before it is sent.
For example, a command like ``{buildinfo: 1}`` can be sent
using:
>>> db.command("buildinfo")
For a command where the value matters, like ``{collstats:
collection_name}`` we can do:
>>> db.command("collstats", collection_name)
For commands that take additional arguments we can use
kwargs. So ``{filemd5: object_id, root: file_root}`` becomes:
>>> db.command("filemd5", object_id, root=file_root)
:Parameters:
- `command`: document representing the command to be issued,
or the name of the command (for simple commands only).
.. note:: the order of keys in the `command` document is
significant (the "verb" must come first), so commands
which require multiple keys (e.g. `findandmodify`)
should use an instance of :class:`~pymongo.son.SON` or
a string and kwargs instead of a Python `dict`.
- `value` (optional): value to use for the command verb when
`command` is passed as a string
- `check` (optional): check the response for errors, raising
:class:`~pymongo.errors.OperationFailure` if there are any
- `allowable_errors`: if `check` is ``True``, error messages
in this list will be ignored by error-checking
- `**kwargs` (optional): additional keyword arguments will
be added to the command document before it is sent
.. versionchanged:: 1.6
Added the `value` argument for string commands, and keyword
arguments for additional command options.
.. versionchanged:: 1.5
`command` can be a string in addition to a full document.
.. versionadded:: 1.4
.. mongodoc:: commands
"""
if isinstance(command, basestring):
command = SON([(command, value)])
command.update(kwargs)
result = self["$cmd"].find_one(command, _sock=_sock,
_must_use_master=True,
_is_command=True)
if check and result["ok"] != 1:
if result["errmsg"] in allowable_errors:
return result
raise OperationFailure("command %r failed: %s" %
(command, result["errmsg"]))
return result
