def get_last_version(self, filename):
"""Get a file from GridFS by ``"filename"``.
Returns the most recently uploaded file in GridFS with the
name `filename` as an instance of
:class:`~gridfs.grid_file.GridOut`. Raises
:class:`~gridfs.errors.NoFile` if no such file exists.
An index on ``{filename: 1, uploadDate: -1}`` will
automatically be created when this method is called the first
time.
:Parameters:
- `filename`: ``"filename"`` of the file to get
.. versionadded:: 1.6
"""
self.__files.ensure_index(
filter.sort(ASCENDING("filename") + DESCENDING("uploadDate")))
doc = yield self.__files.find_one({"filename": filename},
filter=filter.sort(
DESCENDING("uploadDate")))
if doc is None:
raise NoFile("TxMongo: no file in gridfs with filename {0}".format(
repr(filename)))
defer.returnValue(GridOut(self.__collection, doc))
def test_index_info(self):
db = self.db
yield db.test.drop_indexes()
yield db.test.remove({})
db.test.save({}) # create collection
ix_info = yield db.test.index_information()
self.assertEqual(len(ix_info), 1)
self.assertEqual(ix_info["_id_"]["name"], "_id_")
yield db.test.create_index(qf.sort(qf.ASCENDING("hello")))
ix_info = yield db.test.index_information()
self.assertEqual(len(ix_info), 2)
self.assertEqual(ix_info["hello_1"]["name"], "hello_1")
yield db.test.create_index(
qf.sort(qf.DESCENDING("hello") + qf.ASCENDING("world")),
unique=True,
sparse=True)
ix_info = yield db.test.index_information()
self.assertEqual(ix_info["hello_1"]["name"], "hello_1")
self.assertEqual(len(ix_info), 3)
self.assertEqual({
"hello": -1,
"world": 1
}, ix_info["hello_-1_world_1"]["key"])
self.assertEqual(True, ix_info["hello_-1_world_1"]["unique"])
self.assertEqual(True, ix_info["hello_-1_world_1"]["sparse"])
yield db.test.drop_indexes()
yield db.test.remove({})
def example():
mongo = yield txmongo.MongoConnection()
foo = mongo.foo # `foo` database
test = foo.test # `test` collection
idx = filter.sort(filter.ASCENDING("something") + filter.DESCENDING("else"))
print "IDX:", idx
result = yield test.create_index(idx)
print "create_index:", result
result = yield test.index_information()
print "index_information:", result
result = yield test.drop_index(idx)
print "drop_index:", result
# Geohaystack example
geoh_idx = filter.sort(filter.GEOHAYSTACK("loc") + filter.ASCENDING("type"))
print "IDX:", geoh_idx
result = yield test.create_index(geoh_idx, **{'bucketSize':1})
print "index_information:", result
result = yield test.drop_index(geoh_idx)
print "drop_index:", result
# 2D geospatial index
geo_idx = filter.sort(filter.GEO2D("pos"))
print "IDX:", geo_idx
result = yield test.create_index(geo_idx, **{ 'min':-100, 'max':100 })
print "index_information:", result
result = yield test.drop_index(geo_idx)
print "drop_index:", result
def test_index_info(self):
db = self.db
yield db.test.drop_indexes()
yield db.test.remove({})
db.test.save({}) # create collection
ix_info = yield db.test.index_information()
self.assertEqual(len(ix_info), 1)
self.assertEqual(ix_info["_id_"]["name"], "_id_")
yield db.test.create_index(filter.sort(filter.ASCENDING("hello")))
ix_info = yield db.test.index_information()
self.assertEqual(len(ix_info), 2)
self.assertEqual(ix_info["hello_1"]["name"], "hello_1")
yield db.test.create_index(filter.sort(filter.DESCENDING("hello") +
filter.ASCENDING("world")),
unique=True, sparse=True)
ix_info = yield db.test.index_information()
self.assertEqual(ix_info["hello_1"]["name"], "hello_1")
self.assertEqual(len(ix_info), 3)
self.assertEqual({"hello": -1, "world": 1}, ix_info["hello_-1_world_1"]["key"])
self.assertEqual(True, ix_info["hello_-1_world_1"]["unique"])
self.assertEqual(True, ix_info["hello_-1_world_1"]["sparse"])
yield db.test.drop_indexes()
yield db.test.remove({})
def get_last_version(self, filename):
"""Get a file from GridFS by ``"filename"``.
Returns the most recently uploaded file in GridFS with the
name `filename` as an instance of
:class:`~gridfs.grid_file.GridOut`. Raises
:class:`~gridfs.errors.NoFile` if no such file exists.
An index on ``{filename: 1, uploadDate: -1}`` will
automatically be created when this method is called the first
time.
:Parameters:
- `filename`: ``"filename"`` of the file to get
.. versionadded:: 1.6
"""
self.__files.ensure_index(filter.sort(ASCENDING("filename") + DESCENDING("uploadDate")))
doc = yield self.__files.find_one({"filename": filename},
filter=filter.sort(DESCENDING("uploadDate")))
if doc is None:
raise NoFile("TxMongo: no file in gridfs with filename {0}".format(repr(filename)))
defer.returnValue(GridOut(self.__collection, doc))
def test_index_info(self):
db = self.db
yield db.test.drop_indexes()
yield db.test.remove({})
db.test.save({}) # create collection
ix_info = yield db.test.index_information()
self.assertEqual(len(ix_info), 1)
self.assert_("_id_" in ix_info)
yield db.test.create_index(filter.sort(filter.ASCENDING("hello")))
ix_info = yield db.test.index_information()
self.assertEqual(len(ix_info), 2)
self.assertEqual(ix_info["hello_1"], [("hello", 1)])
yield db.test.create_index(filter.sort(filter.DESCENDING("hello") + filter.ASCENDING("world")), unique=True)
ix_info = yield db.test.index_information()
self.assertEqual(ix_info["hello_1"], [("hello", 1)])
self.assertEqual(len(ix_info), 3)
self.assertEqual([("world", 1), ("hello", -1)], ix_info["hello_-1_world_1"])
# Unique key will not show until index_information is updated with changes introduced in version 1.7
#self.assertEqual(True, ix_info["hello_-1_world_1"]["unique"])
yield db.test.drop_indexes()
yield db.test.remove({})
def __init__(self, database, collection="fs"):
"""Create a new instance of :class:`GridFS`.
Raises :class:`TypeError` if `database` is not an instance of
:class:`~pymongo.database.Database`.
:Parameters:
- `database`: database to use
- `collection` (optional): root collection to use
.. note::
Instantiating a GridFS object will implicitly create it indexes.
This could leads to errors if the underlaying connection is closed
before the indexes creation request has returned. To avoid this you
should use the defer returned by :meth:`GridFS.indexes_created`.
.. versionadded:: 1.6
The `collection` parameter.
"""
if not isinstance(database, Database):
raise TypeError("TxMongo: database must be an instance of Database.")
self.__database = database
self.__collection = database[collection]
self.__files = self.__collection.files
self.__chunks = self.__collection.chunks
self.__indexes_created_defer = defer.DeferredList([
self.__files.create_index(
filter.sort(ASCENDING("filesname") + ASCENDING("uploadDate"))),
self.__chunks.create_index(
filter.sort(ASCENDING("files_id") + ASCENDING("n")), unique=True)
])
def test_index_info(self):
db = self.db
yield db.test.drop_indexes()
yield db.test.remove({})
db.test.save({}) # create collection
ix_info = yield db.test.index_information()
self.assertEqual(len(ix_info), 1)
self.assert_("_id_" in ix_info)
yield db.test.create_index(filter.sort(filter.ASCENDING("hello")))
ix_info = yield db.test.index_information()
self.assertEqual(len(ix_info), 2)
self.assertEqual(ix_info["hello_1"], [("hello", 1)])
yield db.test.create_index(filter.sort(
filter.DESCENDING("hello") + filter.ASCENDING("world")),
unique=True)
ix_info = yield db.test.index_information()
self.assertEqual(ix_info["hello_1"], [("hello", 1)])
self.assertEqual(len(ix_info), 3)
self.assertEqual([("world", 1), ("hello", -1)],
ix_info["hello_-1_world_1"])
# Unique key will not show until index_information is updated with changes introduced in version 1.7
#self.assertEqual(True, ix_info["hello_-1_world_1"]["unique"])
yield db.test.drop_indexes()
yield db.test.remove({})
def example():
mongo = yield txmongo.MongoConnection()
foo = mongo.foo # `foo` database
test = foo.test # `test` collection
idx = filter.sort(filter.ASCENDING("something") + filter.DESCENDING("else"))
print "IDX:", idx
result = yield test.create_index(idx)
print "create_index:", result
result = yield test.index_information()
print "index_information:", result
result = yield test.drop_index(idx)
print "drop_index:", result
# Geohaystack example
geoh_idx = filter.sort(filter.GEOHAYSTACK("loc") + filter.ASCENDING("type"))
print "IDX:", geoh_idx
result = yield test.create_index(geoh_idx, **{'bucketSize':1})
print "index_information:", result
result = yield test.drop_index(geoh_idx)
print "drop_index:", result
# 2D geospatial index
geo_idx = filter.sort(filter.GEO2D("pos"))
print "IDX:", geo_idx
result = yield test.create_index(geo_idx, **{ 'min':-100, 'max':100 })
print "index_information:", result
result = yield test.drop_index(geo_idx)
print "drop_index:", result
def __init__(self, database, collection="fs"):
"""Create a new instance of :class:`GridFS`.
Raises :class:`TypeError` if `database` is not an instance of
:class:`~pymongo.database.Database`.
:Parameters:
- `database`: database to use
- `collection` (optional): root collection to use
.. note::
Instantiating a GridFS object will implicitly create it indexes.
This could leads to errors if the underlaying connection is closed
before the indexes creation request has returned. To avoid this you
should use the defer returned by :meth:`GridFS.indexes_created`.
.. versionadded:: 1.6
The `collection` parameter.
"""
if not isinstance(database, Database):
raise TypeError(
"TxMongo: database must be an instance of Database.")
self.__database = database
self.__collection = database[collection]
self.__files = self.__collection.files
self.__chunks = self.__collection.chunks
self.__indexes_created_defer = defer.DeferredList([
self.__files.create_index(
filter.sort(ASCENDING("filename") + ASCENDING("uploadDate"))),
self.__chunks.create_index(
filter.sort(ASCENDING("files_id") + ASCENDING("n")),
unique=True)
])
def test_Sort(self):
doc = yield self.coll.find_one_and_delete({'x': {"$exists": True}},
sort=qf.sort(qf.ASCENDING('y')))
self.assertEqual(doc['x'], 1)
doc = yield self.coll.find_one_and_delete({'x': {"$exists": True}},
sort=qf.sort(qf.DESCENDING('y')))
self.assertEqual(doc['x'], 3)
cnt = yield self.coll.count()
self.assertEqual(cnt, 1)
def test_Sort(self):
doc = yield self.coll.find_one_and_delete({'x': {"$exists": True}},
sort=qf.sort(qf.ASCENDING('y')))
self.assertEqual(doc['x'], 1)
doc = yield self.coll.find_one_and_delete({'x': {"$exists": True}},
sort=qf.sort(qf.DESCENDING('y')))
self.assertEqual(doc['x'], 3)
cnt = yield self.coll.count()
self.assertEqual(cnt, 1)
def test_FilterMerge(self):
self.assertEqual(qf.sort(qf.ASCENDING('x') + qf.DESCENDING('y')),
qf.sort(qf.ASCENDING('x')) + qf.sort(qf.DESCENDING('y')))
comment = "hello world"
yield self.db.command("profile", 2)
yield self.coll.find({}, filter=qf.sort(qf.ASCENDING('x')) + qf.comment(comment))
yield self.db.command("profile", 0)
cnt = yield self.db.system.profile.count({"query.$orderby.x": 1,
"query.$comment": comment})
self.assertEqual(cnt, 1)
def test_Sort(self):
doc = yield self.coll.find_one_and_replace({}, {'x': 5, 'y': 5},
projection={"_id": 0},
sort=qf.sort(qf.ASCENDING('y')))
self.assertEqual(doc, {'x': 10, 'y': 10})
doc = yield self.coll.find_one_and_replace({}, {'x': 40, 'y': 40},
projection={"_id": 0},
sort=qf.sort(qf.DESCENDING('y')))
self.assertEqual(doc, {'x': 30, 'y': 30})
ys = yield self.coll.distinct('y')
self.assertEqual(set(ys), set([5, 20, 40]))
def test_Sort(self):
doc = yield self.coll.find_one_and_update({}, {"$set": {'y': 5}},
projection={"_id": 0},
sort=qf.sort(qf.ASCENDING('y')))
self.assertEqual(doc, {'x': 10, 'y': 10})
doc = yield self.coll.find_one_and_update({}, {"$set": {'y': 35}},
projection={"_id": 0},
sort=qf.sort(qf.DESCENDING('y')))
self.assertEqual(doc, {'x': 30, 'y': 30})
ys = yield self.coll.distinct('y')
self.assertEqual(set(ys), {5, 20, 35})
def test_Sort(self):
doc = yield self.coll.find_one_and_update({}, {"$set": {'y': 5}},
projection={"_id": 0},
sort=qf.sort(qf.ASCENDING('y')))
self.assertEqual(doc, {'x': 10, 'y': 10})
doc = yield self.coll.find_one_and_update({}, {"$set": {'y': 35}},
projection={"_id": 0},
sort=qf.sort(qf.DESCENDING('y')))
self.assertEqual(doc, {'x': 30, 'y': 30})
ys = yield self.coll.distinct('y')
self.assertEqual(set(ys), {5, 20, 35})
def test_index_haystack(self):
db = self.db
coll = self.coll
yield coll.drop_indexes()
_id = yield coll.insert({
"pos": {"long": 34.2, "lat": 33.3},
"type": "restaurant"
})
yield coll.insert({
"pos": {"long": 34.2, "lat": 37.3}, "type": "restaurant"
})
yield coll.insert({
"pos": {"long": 59.1, "lat": 87.2}, "type": "office"
})
yield coll.create_index(filter.sort(filter.GEOHAYSTACK("pos") +
filter.ASCENDING("type")), **{"bucket_size": 1})
results = yield db.command("geoSearch", "mycol",
near=[33, 33],
maxDistance=6,
search={"type": "restaurant"},
limit=30)
self.assertEqual(2, len(results["results"]))
self.assertEqual({
"_id": _id,
"pos": {"long": 34.2, "lat": 33.3},
"type": "restaurant"
}, results["results"][0])
def test_index_haystack(self):
db = self.db
coll = self.coll
yield coll.drop_indexes()
_id = yield coll.insert({
"pos": {"long": 34.2, "lat": 33.3},
"type": "restaurant"
})
yield coll.insert({
"pos": {"long": 34.2, "lat": 37.3}, "type": "restaurant"
})
yield coll.insert({
"pos": {"long": 59.1, "lat": 87.2}, "type": "office"
})
yield coll.create_index(filter.sort(filter.GEOHAYSTACK("pos") + filter.ASCENDING("type")), **{'bucket_size': 1})
# TODO: A db.command method has not been implemented yet.
# Sending command directly
command = SON([
("geoSearch", "mycol"),
("near", [33, 33]),
("maxDistance", 6),
("search", {"type": "restaurant"}),
("limit", 30),
])
results = yield db["$cmd"].find_one(command)
self.assertEqual(2, len(results['results']))
self.assertEqual({
"_id": _id,
"pos": {"long": 34.2, "lat": 33.3},
"type": "restaurant"
}, results["results"][0])
def process_message(self, msg):
headers = 'headers' in msg.content.properties and \
msg.content.properties['headers'] or None
data = msg.content.body
log.msg('RECEIVED MSG: {}'.format(str(headers)))
log.msg(data)
part = {}
for key, value in self.run(data, headers):
ap = part.get(key, None)
if ap is None:
ap = []
part[key] = ap
ap.append((key, value))
data = dict(**headers)
data['result'] = json.dumps(part)
yield self.results.insert(SON(data))
# TODO: I think this is not right place for this code. @german
idx = filter.sort(filter.ASCENDING("task"))
self.results.ensure_index(idx)
headers['worker'] = 'map'
self.publisher.sendMessage('OK',
routing_key=ns.MASTER_QUEUE,
headers=headers)
returnValue(msg)
def test_FilterMerge(self):
self.assertEqual(qf.sort(qf.ASCENDING('x') + qf.DESCENDING('y')),
qf.sort(qf.ASCENDING('x')) + qf.sort(qf.DESCENDING('y')))
comment = "hello world"
yield self.db.command("profile", 2)
yield self.coll.find({}, filter=qf.sort(qf.ASCENDING('x')) + qf.comment(comment))
yield self.db.command("profile", 0)
if (yield self.__3_2_or_higher()):
profile_filter = {"query.sort.x": 1, "query.comment": comment}
else:
profile_filter = {"query.$orderby.x": 1, "query.$comment": comment}
cnt = yield self.db.system.profile.count(profile_filter)
self.assertEqual(cnt, 1)
def process_message(self, msg):
headers = 'headers' in msg.content.properties and \
msg.content.properties['headers'] or None
data = msg.content.body
log.msg('RECEIVED MSG: {}'.format(str(headers)))
log.msg(data)
part = {}
for key, value in self.run(data, headers):
ap = part.get(key, None)
if ap is None:
ap = []
part[key] = ap
ap.append((key, value))
data = dict(**headers)
data['result'] = json.dumps(part)
yield self.results.insert(SON(data))
# TODO: I think this is not right place for this code. @german
idx = filter.sort(filter.ASCENDING("task"))
self.results.ensure_index(idx)
headers['worker'] = 'map'
self.publisher.sendMessage('OK',
routing_key=ns.MASTER_QUEUE,
headers=headers)
returnValue(msg)
def test_index_geo2dsphere(self):
coll = self.coll
geo_ix = yield coll.create_index(qf.sort(qf.GEO2DSPHERE("loc")))
self.assertEqual("loc_2dsphere", geo_ix)
index_info = yield coll.index_information()
self.assertEqual(index_info["loc_2dsphere"]["key"], {"loc": "2dsphere"})
def test_index_geo2d(self):
coll = self.coll
geo_ix = yield coll.create_index(qf.sort(qf.GEO2D("loc")))
self.assertEqual("loc_2d", geo_ix)
index_info = yield coll.index_information()
self.assertEqual({"loc": "2d"}, index_info["loc_2d"]["key"])
def test_create_index_nodup(self):
coll = self.coll
yield coll.insert({'b': 1})
yield coll.insert({'b': 1})
ix = coll.create_index(qf.sort(qf.ASCENDING("b")), unique=True)
yield self.assertFailure(ix, errors.DuplicateKeyError)
def test_index_geo2d(self):
coll = self.coll
geo_ix = yield coll.create_index(qf.sort(qf.GEO2D("loc")))
self.assertEqual("loc_2d", geo_ix)
index_info = yield coll.index_information()
self.assertEqual({"loc": "2d"}, index_info["loc_2d"]["key"])
def test_index_text(self):
ix = yield self.coll.create_index(qf.sort(qf.TEXT("title") + qf.TEXT("summary")),
weights={"title": 100, "summary": 20})
self.assertEqual("title_text_summary_text", ix)
index_info = yield self.coll.index_information()
self.assertEqual(index_info[ix]["key"], {"_fts": "text", "_ftsx": 1})
self.assertEqual(index_info[ix]["weights"], {"title": 100, "summary": 20})
def evaluateIndex(self, indxs):
if ('EmailIndex' in indxs) and (indxs['EmailIndex']['unique'] == True):
return
else:
logging.info("Creating email index...")
self.collection.create_index(
qf.sort(qf.ASCENDING('Email')),
name="EmailIndex", unique=True)
def open_spider(self, spider):
self._db_client = yield ConnectionPool(self.db_uri)
self._db = self._db_client[self.db_name]
self._coll = self._db[self.coll_name]
yield self._coll.find_one(timeout=True)
for index in self.db_index:
yield self._coll.create_index(qf.sort(index))
logger.info('{storage} opened'.format(storage=self.__class__.__name__))
def test_Failures(self):
# can't alter _id
yield self.assertFailure(self.coll.update_many({}, {"$set": {"_id": 1}}), WriteError)
# invalid field name
yield self.assertFailure(self.coll.update_many({}, {"$set": {'$': 1}}), WriteError)
yield self.coll.create_index(qf.sort(qf.ASCENDING('x')), unique=True)
yield self.assertFailure(self.coll.update_many({'x': 2}, {"$set": {'x': 1}}), DuplicateKeyError)
def test_create_index_nodup(self):
coll = self.coll
yield coll.insert({"b": 1})
yield coll.insert({"b": 1})
ix = coll.create_index(qf.sort(qf.ASCENDING("b")), unique=True)
yield self.assertFailure(ix, errors.DuplicateKeyError)
def test_Failures(self):
# can't alter _id
yield self.assertFailure(self.coll.update_many({}, {"$set": {"_id": 1}}), WriteError)
# invalid field name
yield self.assertFailure(self.coll.update_many({}, {"$set": {'$': 1}}), WriteError)
yield self.coll.create_index(qf.sort(qf.ASCENDING('x')), unique=True)
yield self.assertFailure(self.coll.update_many({'x': 2}, {"$set": {'x': 1}}), DuplicateKeyError)
def test_index_geo2dsphere(self):
coll = self.coll
geo_ix = yield coll.create_index(qf.sort(qf.GEO2DSPHERE("loc")))
self.assertEqual("loc_2dsphere", geo_ix)
index_info = yield coll.index_information()
self.assertEqual(index_info["loc_2dsphere"]["key"],
{"loc": "2dsphere"})
def test_ensure_index(self):
db = self.db
coll = self.coll
yield coll.ensure_index(qf.sort(qf.ASCENDING("hello")))
indices = yield coll.index_information()
self.assert_(u"hello_1" in indices)
yield coll.drop_indexes()
def test_FilterMerge(self):
self.assertEqual(
qf.sort(qf.ASCENDING('x') + qf.DESCENDING('y')),
qf.sort(qf.ASCENDING('x')) + qf.sort(qf.DESCENDING('y')))
comment = "hello world"
yield self.db.command("profile", 2)
yield self.coll.find({},
filter=qf.sort(qf.ASCENDING('x')) +
qf.comment(comment))
yield self.db.command("profile", 0)
cnt = yield self.db.system.profile.count({
"query.$orderby.x": 1,
"query.$comment": comment
})
self.assertEqual(cnt, 1)
def test_create_index_nodup(self):
coll = self.coll
coll.drop()
coll.insert({'b': 1})
coll.insert({'b': 1})
ix = coll.create_index(filter.sort(filter.ASCENDING("b")), unique=True)
return self.assertFailure(ix, errors.DuplicateKeyError)
def render_GET(self, request):
def handle_posts(posts):
context = {'posts': posts}
request.write(render_response('posts.html', context))
request.finish()
f = _filter.sort(_filter.DESCENDING('date'))
deferred = _db.posts.find(filter=f)
deferred.addCallback(handle_posts)
return NOT_DONE_YET
def ensure_indexes(cls):
"""
Check&create if needed the Document's indexes in database
"""
for index in cls.opts.indexes:
kwargs = index.document.copy()
keys = kwargs.pop('key')
index = qf.sort([(k, d) for k, d in keys.items()])
yield cls.collection.create_index(index, **kwargs)
def ensure_indexes(cls):
"""
Check&create if needed the Document's indexes in database
"""
for index in cls.opts.indexes:
kwargs = index.document.copy()
keys = kwargs.pop('key')
index = qf.sort([(k, d) for k, d in keys.items()])
yield cls.collection.create_index(index, **kwargs)
def test_Failures(self):
yield self.assertFailure(
self.coll.replace_one({'x': 1}, {'x': {
'$': 5
}}), WriteError)
yield self.coll.create_index(qf.sort(qf.ASCENDING('x')), unique=True)
yield self.assertFailure(self.coll.replace_one({'x': 1}, {'x': 2}),
DuplicateKeyError)
def test_ensure_index(self):
db = self.db
coll = self.coll
yield coll.ensure_index(qf.sort(qf.ASCENDING("hello")))
indices = yield db.system.indexes.find({"ns": u"mydb.mycol"})
self.assert_(u"hello_1" in [a["name"] for a in indices])
yield coll.drop_indexes()
def test_ensure_index(self):
db = self.db
coll = self.coll
yield coll.ensure_index(filter.sort(filter.ASCENDING("hello")))
indices = yield db.system.indexes.find({"ns": u"mydb.mycol"})
self.assert_(u"hello_1" in [a["name"] for a in indices])
yield coll.drop_indexes()
def sendHistory(self):
hist = yield self.settings.coll.find(
filter=qf.sort(qf.DESCENDING('t')), limit=10)
for row in hist:
row['uri'] = row.pop('_id')
row['t'] = row['t'].replace(tzinfo=tzutc()).isoformat()
hist.reverse()
self.sendMessage({'type': 'history', 'you': self.agent, 'history': hist})
def test_create_index_nodup(self):
coll = self.coll
ret = yield coll.drop()
ret = yield coll.insert({'b': 1})
ret = yield coll.insert({'b': 1})
ix = coll.create_index(filter.sort(filter.ASCENDING("b")), unique=True)
yield self.assertFailure(ix, errors.DuplicateKeyError)
def create_index(self, spider: Spider):
results = []
for field, _order, *args in self.settings.get(MONGODB_INDEXES, list()):
try:
_ = yield self.coll.create_index(txfilter.sort(_order(field)),
**args[0])
results.append(_)
except OperationFailure:
pass
return results
def test_FilterMerge(self):
self.assertEqual(
qf.sort(qf.ASCENDING('x') + qf.DESCENDING('y')),
qf.sort(qf.ASCENDING('x')) + qf.sort(qf.DESCENDING('y')))
comment = "hello world"
yield self.db.command("profile", 2)
yield self.coll.find({},
filter=qf.sort(qf.ASCENDING('x')) +
qf.comment(comment))
yield self.db.command("profile", 0)
if (yield self.__3_2_or_higher()):
profile_filter = {"query.sort.x": 1, "query.comment": comment}
else:
profile_filter = {"query.$orderby.x": 1, "query.$comment": comment}
cnt = yield self.db.system.profile.count(profile_filter)
self.assertEqual(cnt, 1)
def test_index_geo2d(self):
db = self.db
coll = self.coll
yield coll.drop_indexes()
geo_ix = yield coll.create_index(filter.sort(filter.GEO2D("loc")))
self.assertEqual('loc_2d', geo_ix)
index_info = yield coll.index_information()
self.assertEqual([('loc', '2d')], index_info['loc_2d'])
def test_index_geo2d(self):
db = self.db
coll = self.coll
yield coll.drop_indexes()
geo_ix = yield coll.create_index(filter.sort(filter.GEO2D("loc")))
self.assertEqual('loc_2d', geo_ix)
index_info = yield coll.index_information()
self.assertEqual([('loc', '2d')], index_info['loc_2d'])
def test_create_index(self):
db = self.db
coll = self.coll
self.assertRaises(TypeError, coll.create_index, 5)
self.assertRaises(TypeError, coll.create_index, {"hello": 1})
yield coll.insert({'c': 1}) # make sure collection exists.
yield coll.drop_indexes()
count = len((yield coll.index_information()))
self.assertEqual(count, 1)
self.assertIsInstance(count, int)
yield coll.create_index(qf.sort(qf.ASCENDING("hello")))
yield coll.create_index(
qf.sort(qf.ASCENDING("hello") + qf.DESCENDING("world")))
count = len((yield coll.index_information()))
self.assertEqual(count, 3)
yield coll.drop_indexes()
ix = yield coll.create_index(
qf.sort(qf.ASCENDING("hello") + qf.DESCENDING("world")),
name="hello_world")
self.assertEquals(ix, "hello_world")
yield coll.drop_indexes()
count = len((yield coll.index_information()))
self.assertEqual(count, 1)
yield coll.create_index(qf.sort(qf.ASCENDING("hello")))
indices = yield coll.index_information()
self.assert_(u"hello_1" in indices)
yield coll.drop_indexes()
count = len((yield coll.index_information()))
self.assertEqual(count, 1)
ix = yield coll.create_index(
qf.sort(qf.ASCENDING("hello") + qf.DESCENDING("world")))
self.assertEquals(ix, "hello_1_world_-1")
def test_create_index(self):
db = self.db
coll = self.coll
self.assertFailure(coll.create_index(5), TypeError)
self.assertFailure(coll.create_index({"hello": 1}), TypeError)
yield coll.insert({'c': 1}) # make sure collection exists.
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
self.assertIsInstance(count, int)
yield coll.create_index(qf.sort(qf.ASCENDING("hello")))
yield coll.create_index(
qf.sort(qf.ASCENDING("hello") + qf.DESCENDING("world")))
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 3)
yield coll.drop_indexes()
ix = yield coll.create_index(
qf.sort(qf.ASCENDING("hello") + qf.DESCENDING("world")),
name="hello_world")
self.assertEquals(ix, "hello_world")
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
yield coll.create_index(qf.sort(qf.ASCENDING("hello")))
indices = yield db.system.indexes.find({"ns": u"mydb.mycol"})
self.assert_(u"hello_1" in [a["name"] for a in indices])
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
ix = yield coll.create_index(
qf.sort(qf.ASCENDING("hello") + qf.DESCENDING("world")))
self.assertEquals(ix, "hello_1_world_-1")
def test_create_index_dropdups(self):
# dropDups was removed from MongoDB v3.0
ismaster = yield self.db.command("ismaster")
if ismaster["maxWireVersion"] >= 3:
raise unittest.SkipTest("dropDups was removed from MongoDB 3")
yield self.coll.insert([{"b": 1}, {"b": 1}])
yield self.coll.create_index(qf.sort(qf.ASCENDING("b")), unique=True, drop_dups=True)
docs = yield self.coll.find(fields={"_id": 0})
self.assertEqual(docs, [{"b": 1}])
def test_index_haystack(self):
db = self.db
coll = self.coll
yield coll.drop_indexes()
_id = yield coll.insert({
"pos": {
"long": 34.2,
"lat": 33.3
},
"type": "restaurant"
})
yield coll.insert({
"pos": {
"long": 34.2,
"lat": 37.3
},
"type": "restaurant"
})
yield coll.insert({
"pos": {
"long": 59.1,
"lat": 87.2
},
"type": "office"
})
yield coll.create_index(
filter.sort(filter.GEOHAYSTACK("pos") + filter.ASCENDING("type")),
**{'bucket_size': 1})
# TODO: A db.command method has not been implemented yet.
# Sending command directly
command = SON([
("geoSearch", "mycol"),
("near", [33, 33]),
("maxDistance", 6),
("search", {
"type": "restaurant"
}),
("limit", 30),
])
results = yield db["$cmd"].find_one(command)
self.assertEqual(2, len(results['results']))
self.assertEqual(
{
"_id": _id,
"pos": {
"long": 34.2,
"lat": 33.3
},
"type": "restaurant"
}, results["results"][0])
def open_spider(self, spider):
# Sync
# self.client = pymongo.MongoClient(self.settings['MONGODB_URI'])
# self.db = self.client[self.settings['MONGODB_DB']]
# self.coll = self.db[self.settings['MONGODB_COLL_RAW']]
# self.coll.create_index('request_url')
# Async
self.client = yield ConnectionPool(self.settings['MONGODB_URI'])
self.db = self.client[self.settings['MONGODB_DB']]
self.coll = self.db[self.settings['MONGODB_COLL_RAW']]
self.coll.create_index(sort([('request_url', 1)]))
def test_create_index(self):
db = self.db
coll = self.coll
self.assertRaises(TypeError, coll.create_index, 5)
self.assertRaises(TypeError, coll.create_index, {"hello": 1})
yield coll.insert({'c': 1}) # make sure collection exists.
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
result1 = yield coll.create_index(
filter.sort(filter.ASCENDING("hello")))
result2 = yield coll.create_index(filter.sort(filter.ASCENDING("hello") + \
filter.DESCENDING("world")))
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 3)
yield coll.drop_indexes()
ix = yield coll.create_index(filter.sort(filter.ASCENDING("hello") + \
filter.DESCENDING("world")), name="hello_world")
self.assertEquals(ix, "hello_world")
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
yield coll.create_index(filter.sort(filter.ASCENDING("hello")))
indices = yield db.system.indexes.find({"ns": u"mydb.mycol"})
self.assert_(u"hello_1" in [a["name"] for a in indices])
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
ix = yield coll.create_index(filter.sort(filter.ASCENDING("hello") + \
filter.DESCENDING("world")))
self.assertEquals(ix, "hello_1_world_-1")
def test_create_index(self):
db = self.db
coll = self.coll
self.assertFailure(coll.create_index(5), TypeError)
self.assertFailure(coll.create_index({"hello": 1}), TypeError)
yield coll.insert({'c': 1}) # make sure collection exists.
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
self.assertIsInstance(count, int)
yield coll.create_index(qf.sort(qf.ASCENDING("hello")))
yield coll.create_index(qf.sort(qf.ASCENDING("hello") +
qf.DESCENDING("world")))
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 3)
yield coll.drop_indexes()
ix = yield coll.create_index(qf.sort(qf.ASCENDING("hello") +
qf.DESCENDING("world")), name="hello_world")
self.assertEquals(ix, "hello_world")
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
yield coll.create_index(qf.sort(qf.ASCENDING("hello")))
indices = yield db.system.indexes.find({"ns": u"mydb.mycol"})
self.assert_(u"hello_1" in [a["name"] for a in indices])
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
ix = yield coll.create_index(qf.sort(qf.ASCENDING("hello") +
qf.DESCENDING("world")))
self.assertEquals(ix, "hello_1_world_-1")
def get_msg_list(cls, user_id, chatter_id, start_time = None, end_time = None):
query_spec = {'$or': [{'send_id': user_id, 'receive_id': chatter_id},
{'send_id': chatter_id, 'receive_id': user_id}]}
if start_time or end_time:
if start_time:
time_query = {'$gte': start_time}
if end_time:
time_query = {'$lte': end_time}
query_spec.update({'send_time': time_query})
msg_list = yield cls.collection.find(query_spec, filter = query_filter.sort([('send_time', -1)]))
defer.returnValue(cls.serialize(msg_list))
def test_drop_index(self):
index = qf.sort(qf.ASCENDING("hello") + qf.DESCENDING("world"))
yield self.coll.create_index(index, name="myindex")
res = yield self.coll.drop_index("myindex")
self.assertEqual(res["ok"], 1)
yield self.coll.create_index(index)
res = yield self.coll.drop_index(index)
self.assertEqual(res["ok"], 1)
self.assertRaises(TypeError, self.coll.drop_index, 123)
def test_drop_index(self):
index = qf.sort(qf.ASCENDING("hello") + qf.DESCENDING("world"))
yield self.coll.create_index(index, name="myindex")
res = yield self.coll.drop_index("myindex")
self.assertEqual(res["ok"], 1)
yield self.coll.create_index(index)
res = yield self.coll.drop_index(index)
self.assertEqual(res["ok"], 1)
self.assertRaises(TypeError, self.coll.drop_index, 123)
def test_create_index(self):
db = self.db
coll = self.coll
self.assertRaises(TypeError, coll.create_index, 5)
self.assertRaises(TypeError, coll.create_index, {"hello": 1})
yield coll.insert({'c': 1}) # make sure collection exists.
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
result1 = yield coll.create_index(filter.sort(filter.ASCENDING("hello")))
result2 = yield coll.create_index(filter.sort(filter.ASCENDING("hello") + \
filter.DESCENDING("world")))
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 3)
yield coll.drop_indexes()
ix = yield coll.create_index(filter.sort(filter.ASCENDING("hello") + \
filter.DESCENDING("world")), name="hello_world")
self.assertEquals(ix, "hello_world")
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
yield coll.create_index(filter.sort(filter.ASCENDING("hello")))
indices = yield db.system.indexes.find({"ns": u"mydb.mycol"})
self.assert_(u"hello_1" in [a["name"] for a in indices])
yield coll.drop_indexes()
count = yield db.system.indexes.count({"ns": u"mydb.mycol"})
self.assertEqual(count, 1)
ix = yield coll.create_index(filter.sort(filter.ASCENDING("hello") + \
filter.DESCENDING("world")))
self.assertEquals(ix, "hello_1_world_-1")
def test_hint(self):
yield self.coll.create_index(qf.sort(qf.ASCENDING('x')))
yield self.db.command("profile", 2)
cnt = yield self.coll.count(hint=qf.hint(qf.ASCENDING('x')))
self.assertEqual(cnt, 3)
yield self.db.command("profile", 0)
cmds = yield self.db.system.profile.count({"command.hint": {"x": 1}})
self.assertEqual(cmds, 1)
self.assertRaises(TypeError, self.coll.count, hint={'x': 1})
self.assertRaises(TypeError, self.coll.count, hint=[('x', 1)])
