def test_GridFsObjects(self):
""" Tests gridfs objects """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
db.fs.files.remove({}) # drop all objects there first
db.fs.chunks.remove({})
gfs = GridFS(db) # Default collection
yield gfs.delete(u"test")
_ = gfs.new_file(filename="test_1", contentType="text/plain", chunk_size=65536)
yield conn.disconnect()
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
db.fs.files.remove({}) # drop all objects there first
gfs = GridFS(db) # Default collection
_ = yield gfs.put(b"0xDEADBEEF", filename="test_2", contentType="text/plain",
chunk_size=65536)
# disconnect
yield conn.disconnect()
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
gfs = GridFS(db) # Default collection
_ = yield gfs.get("test_3")
# disconnect
yield conn.disconnect()
def test_GridFsObjects(self):
""" Tests gridfs objects """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
gfs = GridFS(db) # Default collection
yield gfs.delete(u"test")
_ = gfs.new_file(filename="test_1",
contentType="text/plain",
chunk_size=65536)
yield conn.disconnect()
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
gfs = GridFS(db) # Default collection
_ = yield gfs.put(b"0xDEADBEEF",
filename="test_2",
contentType="text/plain",
chunk_size=65536)
# disconnect
yield conn.disconnect()
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
gfs = GridFS(db) # Default collection
# Missing file raises error
yield self.assertFailure(gfs.get("test_3"), NoFile)
# disconnect
yield conn.disconnect()
def test_GridFsIndexesCreation(self):
""" Tests gridfs indexes creation"""
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
# Create a new GridFS instance should trigger indexes creation
gfs = GridFS(db)
# Multiple calls should return multiple defer not to mix between them
self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created())
yield gfs.indexes_created()
indexes = yield db.fs.files.index_information()
self.assertTrue(
any(key["key"] == SON([("filename", 1), ("uploadDate", 1)])
for key in indexes.values()))
indexes = yield db.fs.chunks.index_information()
self.assertTrue(
any(key["key"] == SON([("files_id", 1), ("n", 1)])
for key in indexes.values()))
yield conn.disconnect()
def test_GridFsIteration(self):
""" Tests gridfs iterator """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
db.fs.files.remove({}) # drop all objects there first
db.fs.chunks.remove({})
gfs = GridFS(db) # Default collection
new_file = gfs.new_file(filename="testName", contentType="text/plain", length=1048576,
chunk_size=4096)
yield new_file.write(b"0xDEADBEEF"*4096*2)
yield new_file.close()
fake_doc = {"_id": new_file._id, "name": "testName", "length": 4096*2, "chunkSize": 4096,
"contentType": "text/plain"}
grid_out_file = GridOut(db.fs, fake_doc)
iterator = GridOutIterator(grid_out_file, db.fs.chunks)
next_it = yield next(iterator)
self.assertEqual(len(next_it), 4096)
_ = yield next(iterator)
next_it = yield next(iterator)
self.assertEqual(next_it, None)
fake_bad_doc = {"_id": "bad_id", "name": "testName", "length": 4096*2,
"chunkSize": 4096, "contentType": "text/plain"}
grid_bad_out_file = GridOut(db.fs, fake_bad_doc)
bad_iterator = GridOutIterator(grid_bad_out_file, db.fs.chunks)
if _version.version.major >= 15:
with self.assertRaises(errors.CorruptGridFile):
next_it = yield bad_iterator.next()
# disconnect
yield conn.disconnect()
def __init__(self, mode='test'):
'''
Mode is one of ['test', 'development', 'production']
'''
# create a connection to the mongodb. None passed to trigger right away
self.client = txmongo.MongoConnection(pool_size=100)
self.mode = mode
# Pick the right database
if mode == 'development':
self.db = self.client.development
elif mode == 'production':
self.db = self.client.production
elif mode == 'test':
self.db = self.client.test
else:
raise SyntaxError(
"Mode must be one of ['development', 'production', 'test']. the input "
+ mode + " is invalid.")
# Pull refs for easier access later (and to declare them for anyone reading this)
self.users = self.db.users
self.chutes = self.db.chutes
self.routerInstances = self.db.routerInstances
self.chuteInstances = self.db.chuteInstances
def example():
mongo = yield txmongo.MongoConnection()
foo = mongo.foo # `foo` database
test = foo.test # `test` collection
idx = filter.sort(filter.ASCENDING("something") + filter.DESCENDING("else"))
print "IDX:", idx
result = yield test.create_index(idx)
print "create_index:", result
result = yield test.index_information()
print "index_information:", result
result = yield test.drop_index(idx)
print "drop_index:", result
# Geohaystack example
geoh_idx = filter.sort(filter.GEOHAYSTACK("loc") + filter.ASCENDING("type"))
print "IDX:", geoh_idx
result = yield test.create_index(geoh_idx, **{'bucketSize':1})
print "index_information:", result
result = yield test.drop_index(geoh_idx)
print "drop_index:", result
# 2D geospatial index
geo_idx = filter.sort(filter.GEO2D("pos"))
print "IDX:", geo_idx
result = yield test.create_index(geo_idx, **{ 'min':-100, 'max':100 })
print "index_information:", result
result = yield test.drop_index(geo_idx)
print "drop_index:", result
def example():
mongo = yield txmongo.MongoConnection()
foo = mongo.foo # `foo` database
test = foo.test # `test` collection
yield test.insert({"src": "Twitter", "content": "bla bla"}, safe=True)
yield test.insert({"src": "Twitter", "content": "more data"}, safe=True)
yield test.insert({
"src": "Wordpress",
"content": "blog article 1"
},
safe=True)
yield test.insert({
"src": "Wordpress",
"content": "blog article 2"
},
safe=True)
yield test.insert({
"src": "Wordpress",
"content": "some comments"
},
safe=True)
result = yield test.group(keys=["src"],
initial={"count": 0},
reduce="function(obj,prev){prev.count++;}")
print "result:", result
def test_MongoOperations(self):
""" Tests mongo operations """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
test = conn.foo.test
# insert
doc = {"foo":"bar", "items":[1, 2, 3]}
yield test.insert(doc, safe=True)
result = yield test.find_one(doc)
self.assertEqual(result.has_key("_id"), True)
self.assertEqual(result["foo"], "bar")
self.assertEqual(result["items"], [1, 2, 3])
# insert preserves object id
doc.update({'_id': objectid.ObjectId()})
yield test.insert(doc, safe=True)
result = yield test.find_one(doc)
self.assertEqual(result.get('_id'), doc.get('_id'))
self.assertEqual(result["foo"], "bar")
self.assertEqual(result["items"], [1, 2, 3])
# update
yield test.update({"_id":result["_id"]}, {"$set":{"one":"two"}}, safe=True)
result = yield test.find_one({"_id":result["_id"]})
self.assertEqual(result["one"], "two")
# delete
yield test.remove(result["_id"], safe=True)
# disconnect
yield conn.disconnect()
def handle_request(self, request):
try:
username = request.args['user'][0]
cxn = yield txmongo.MongoConnection()
# Don't yield the user query
user_defer = cxn.pycon.users.find_one({'name': username})
# Go ahead with the translation regardless of
# whether we find a user.
input_str = request.args['data'][0]
from txtranslator import TranslatorClient
client = TranslatorClient(SVC_HOST, SVC_PORT)
output_defer = client.translate2(input_str)
# Wait until we have both results.
output_str = yield output_defer
user = yield user_defer
if user:
request.write("%s: %s" % (str(user['name']), output_str))
else:
request.write("User '%s' not found!" % username)
except Exception, err:
print err
request.write(resource.ErrorPage(500, "Internal Server Error.", err).render(request))
def test_Properties(self):
conn = txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.mydb
coll = db.mycol
try:
# names
self.assertEqual(db.name, u"mydb")
self.assertEqual(coll.name, u"mycol")
self.assertEqual(coll.full_name, u"mydb.mycol")
self.assertEqual(coll.subcoll.name, u"mycol.subcoll")
self.assertEqual(coll.subcoll.full_name, u"mydb.mycol.subcoll")
# database
self.assertTrue(coll.database is db)
# Write concern
w2 = coll.with_options(write_concern=WriteConcern(w=2))
dbw2 = database.Database(conn, "mydb", write_concern=WriteConcern(w=2))
self.assertEqual(w2.write_concern, WriteConcern(w=2))
self.assertEqual(dbw2.write_concern, WriteConcern(w=2))
# Connection
self.assertTrue(db.connection is conn)
finally:
yield conn.disconnect()
def _getConnection(self):
if self._connection:
return self._connection
else:
if self.pool:
return txmongo.MongoConnectionPool()
else:
return txmongo.MongoConnection()
def test_MongoObjects(self):
""" Tests creating mongo objects """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
mydb = conn.mydb
self.assertEqual(isinstance(mydb, database.Database), True)
mycol = mydb.mycol
self.assertEqual(isinstance(mycol, collection.Collection), True)
yield conn.disconnect()
def example():
mongo = yield txmongo.MongoConnection()
foo = mongo.foo # `foo` database
test = foo.test # `test` collection
result = yield test.drop(safe=True)
print result
def test_MongoConnection(self):
# MongoConnection returns deferred, which gets MongoAPI
conn = txmongo.MongoConnection(mongo_host, mongo_port)
self.assertEqual(isinstance(conn, defer.Deferred), True)
rapi = yield conn
self.assertEqual(isinstance(rapi, txmongo.MongoAPI), True)
disconnected = yield rapi.disconnect()
self.assertEqual(disconnected, True)
def test_lazyMongoConnectionPool(self):
# lazyMongoConnection returns MongoAPI
rapi = txmongo.MongoConnection(
hosts=['%s:%d' % (mongo_host, mongo_port)], pool_size=2, lazy=True)
self.assertEqual(isinstance(rapi, txmongo.MongoAPI), True)
yield rapi._connected
disconnected = yield rapi.disconnect()
self.assertEqual(disconnected, True)
def test_GridFileObjects(self):
""" Tests gridfs objects """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
db.fs.files.remove({}) # drop all objects there first
db.fs.chunks.remove({})
self.assertRaises(TypeError, GridFS, None)
_ = GridFS(db) # Default collection
self.assertRaises(TypeError, GridIn, None)
with GridIn(db.fs, filename="test_with", contentType="text/plain", chunk_size=1024):
pass
grid_in_file = GridIn(db.fs, filename="test_1", contentType="text/plain",
content_type="text/plain", chunk_size=65536, length=1048576,
upload_date="20150101")
self.assertFalse(grid_in_file.closed)
if _version.version.major >= 15:
with self.assertRaises(TypeError):
yield grid_in_file.write(1)
with self.assertRaises(TypeError):
yield grid_in_file.write(u"0xDEADBEEF")
with self.assertRaises(AttributeError):
_ = grid_in_file.test
grid_in_file.test = 1
yield grid_in_file.write(b"0xDEADBEEF")
yield grid_in_file.write(b"0xDEADBEEF"*1048576)
fake_doc = {"_id": "test_id", "length": 1048576, "filename": "test",
"upload_date": "20150101"}
self.assertRaises(TypeError, GridOut, None, None)
grid_out_file = GridOut(db.fs, fake_doc)
if _version.version.major >= 15:
with self.assertRaises(AttributeError):
_ = grid_out_file.testing
self.assertEqual("test", grid_out_file.filename)
self.assertEqual(0, grid_out_file.tell())
grid_out_file.seek(1024)
self.assertEqual(1024, grid_out_file.tell())
grid_out_file.seek(1024, os.SEEK_CUR)
self.assertEqual(2048, grid_out_file.tell())
grid_out_file.seek(0, os.SEEK_END)
self.assertEqual(1048576, grid_out_file.tell())
self.assertRaises(IOError, grid_out_file.seek, 0, 4)
self.assertRaises(IOError, grid_out_file.seek, -1)
self.assertTrue("'_id': 'test_id'" in repr(grid_out_file))
self.assertTrue("20150101", grid_in_file.upload_date)
yield grid_in_file.writelines([b"0xDEADBEEF", b"0xDEADBEAF"])
yield grid_in_file.close()
if _version.version.major >= 15:
with self.assertRaises(AttributeError):
grid_in_file.length = 1
self.assertEqual(1, grid_in_file.test)
if _version.version.major >= 15:
with self.assertRaises(AttributeError):
_ = grid_in_file.test_none
self.assertTrue(grid_in_file.closed)
if _version.version.major >= 15:
with self.assertRaises(ValueError):
yield grid_in_file.write(b"0xDEADBEEF")
yield conn.disconnect()
def render_GET(self, request):
error_str = validate_params(request, ('user', 'data'))
if error_str: return error_str
mongo_defer = txmongo.MongoConnection()
mongo_defer.addCallback(mongocxn_callback, request)
mongo_defer.addErrback(mongocxn_errback, request)
return NOT_DONE_YET
def test_MongoConnectionPool(self):
# MongoConnectionPool returns deferred, which gets MongoAPI
conn = txmongo.MongoConnection(
hosts=['%s:%d' % (mongo_host, mongo_port)], pool_size=2)
self.assertEqual(isinstance(conn, defer.Deferred), True)
rapi = yield conn
self.assertEqual(isinstance(rapi, txmongo.MongoAPI), True)
disconnected = yield rapi.disconnect()
self.assertEqual(disconnected, True)
def init_database(self):
self.mongo = yield txmongo.MongoConnection(
host=self.setting.DATA_MONGO_HOST,
port=self.setting.DATA_MONGO_PORT)
self.redis = yield txredisapi.Connection(
host=self.setting.QUEUE_REDIS_HOST,
port=self.setting.QUEUE_REDIS_PORT)
#yield self.mongo.admin.authenticate("root", "chenfuzhi")
defer.returnValue(True)
def main():
mongo = yield txmongo.MongoConnection()
admin = yield mongo.mydb.users.find_one({"u":"cyclone"})
if admin:
print "user 'cyclone' with password 'cyclone' already exists: %s" % admin["_id"]
else:
objid = yield mongo.mydb.users.insert({"u":"cyclone", "p":hashlib.md5("cyclone").hexdigest()}, safe=True)
print "user 'cyclone' with password 'cyclone' has been created: %s" % objid
def example():
mongo = yield txmongo.MongoConnection()
foo = mongo.foo # `foo` database
test = foo.test # `test` collection
# insert some data
for x in xrange(10000):
result = yield test.insert({"something": x * time.time()}, safe=True)
print result
def example():
mongo = yield txmongo.MongoConnection()
foo = mongo.foo # `foo` database
test = foo.test # `test` collection
# fetch some documents
docs = yield test.find(limit=10)
for doc in docs:
print doc
def test_GridFsOperations(self):
""" Tests gridfs operations """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
# Drop files first TODO: iterate through files and delete them
yield self._drop_gridfs(db)
# Don't forget to disconnect
self.addCleanup(self._disconnect, conn)
try:
in_file = StringIO(b"Test input string")
out_file = StringIO()
except Exception as e:
self.fail("Failed to create memory files for testing: %s" % e)
g_out = None
try:
# Tests writing to a new gridfs file
gfs = GridFS(db) # Default collection
if twisted_version.major >= 15:
with self.assertRaises(NoFile):
yield gfs.get_last_version("optest")
g_in = gfs.new_file(
filename="optest", contentType="text/plain",
chunk_size=65536) # non-default chunk size used
# yielding to ensure writes complete before we close and close before we try to read
yield g_in.write(in_file.read())
yield g_in.close()
# Tests reading from an existing gridfs file
g_out = yield gfs.get_last_version("optest")
data = yield g_out.read()
out_file.write(data)
_id = g_out._id
except Exception as e:
self.fail("Failed to communicate with the GridFS. " +
"Is MongoDB running? %s" % e)
else:
self.assertEqual(in_file.getvalue(), out_file.getvalue(),
"Could not read the value from writing an input")
finally:
in_file.close()
out_file.close()
if g_out:
g_out.close()
listed_files = yield gfs.list()
self.assertEqual(
["optest"], listed_files,
"`optest` is the only expected file and we received %s" %
listed_files)
yield gfs.delete(_id)
def test_GridFsOperations(self):
""" Tests gridfs operations """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
collection = db.fs
# Don't forget to disconnect
self.addCleanup(self._disconnect, conn)
try:
in_file = StringIO("Test input string")
out_file = StringIO()
except Exception, e:
self.fail("Failed to create memory files for testing: %s" % e)
def instatll(self):
try:
self.mongo = yield txmongo.MongoConnection(
host=self.setting.DATA_MONGO_HOST,
port=self.setting.DATA_MONGO_PORT)
self.redis = yield txredisapi.Connection(
host=self.setting.QUEUE_REDIS_HOST,
port=self.setting.QUEUE_REDIS_PORT)
yield self.mongo.admin.authenticate("root", "chenfuzhi")
defer.returnValue(True)
except Exception, e:
defer.returnValue(False)
def example():
mongo = yield txmongo.MongoConnection()
foo = mongo.foo # `foo` database
test = foo.test # `test` collection
# create the filter
f = txmongo.filter.sort(txmongo.filter.DESCENDING("something"))
#f += txmongo.filter.hint(txmongo.filter.DESCENDING("myindex"))
#f += txmongo.filter.explain()
# fetch some documents
docs = yield test.find(limit=10, filter=f)
for n, doc in enumerate(docs):
print n, doc
def test_GridFsObjects(self):
""" Tests gridfs objects """
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
collection = db.fs
gfs = gridfs.GridFS(db) # Default collection
gridin = GridIn(collection, filename='test', contentType="text/plain",
chunk_size=2**2**2**2)
new_file = gfs.new_file(filename='test2', contentType="text/plain",
chunk_size=2**2**2**2)
# disconnect
yield conn.disconnect()
def example():
mongo = yield txmongo.MongoConnection()
foo = mongo.foo # `foo` database
test = foo.test # `test` collection
# insert
yield test.insert({"foo": "bar", "name": "bla"}, safe=True)
# update
result = yield test.update({"foo": "bar"}, {"$set": {
"name": "john doe"
}},
safe=True)
print "result:", result
def test_GridFsIndexesCreation(self):
""" Tests gridfs indexes creation"""
conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
db = conn.test
yield self._drop_gridfs(db)
# Create a new GridFS instance should trigger indexes creation
gfs = GridFS(db)
# Multiple calls should return multiple defer not to mix between them
self.assertNotEqual(gfs.indexes_created(), gfs.indexes_created())
yield gfs.indexes_created()
yield conn.disconnect()
def example():
mongo = yield txmongo.MongoConnection()
foo = mongo.foo # `foo` database
test = foo.test # `test` collection
# insert
result = yield test.insert({"foo":"bar", "name":"bla"}, safe=True)
print "insert result:", result
# update
result = yield test.update({"foo":"bar"}, {"$set": {"name":"john doe"}}, safe=True)
print "update result:", result
# find and modify
spec1 = {"query": {"foo":"bar"},
"update": {"$set": {"name": "findAndModify collection"}},
"new": True, }
result = yield test.runCommand("findAndModify", test.name, **spec1)
print "findAndModify via collection.runCommand updated doc:", result
spec2 = {"query": {"foo":"bar"},
"update": {"$set": {"name": "findAndModify database"}},
"new": True, }
result = yield foo.command("findAndModify", test.name, **spec2)
print "findAndModify via database.command updated doc:", result
result = yield test.findAndModify(query = {"foo":"bar"},
update = {"$set": {"name": "findAndModify method"}},
**{"new": True} )
print "findAndModify via collection method updated doc:", result
result = yield test.findAndModify(query = {"foo":"barsert new"},
update = {"$set": {"name": "findAndModify upsert"}},
upsert = True,
**{"new": True} )
print "findAndModify via collection method upserted doc:", result
result = yield test.findAndModify(query = {"foo":"barsert old"},
update = {"$set": {"name": "findAndModify upsert"}},
upsert = True, )
print "findAndModify via collection method result (old doc):", result
result = yield foo.command("profile", -1)
print "profile level:", result
result = yield foo.command("ping")
print "ping:", result
