def test_query_slave(self):
try:
test_shunt.setup()
masterdb = asyncmongo.Client(pool_id='testquerymaster',
host='127.0.0.1',
port=27017,
dbname='test',
maxconnections=2)
slavedb = asyncmongo.Client(pool_id='testqueryslave',
host='127.0.0.1',
port=27018,
dbname='test',
maxconnections=2,
slave_okay=True)
logging.debug(
'waiting for replication to start (sleeping 4 seconds)')
time.sleep(4)
def update_callback(response, error):
tornado.ioloop.IOLoop.instance().stop()
logging.info(response)
assert len(response) == 1
test_shunt.register_called('update')
masterdb.test_stats.update({"_id": TEST_TIMESTAMP},
{'$inc': {
'test_count': 1
}},
upsert=True,
callback=update_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('update')
# wait for the insert to get to the slave
time.sleep(2.5)
def query_callback(response, error):
tornado.ioloop.IOLoop.instance().stop()
logging.info(response)
logging.info(error)
assert error is None
assert isinstance(response, dict)
assert response['_id'] == TEST_TIMESTAMP
assert response['test_count'] == 1
test_shunt.register_called('retrieved')
slavedb.test_stats.find_one({"_id": TEST_TIMESTAMP},
callback=query_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('retrieved')
except:
tornado.ioloop.IOLoop.instance().stop()
raise
def test_duplicate_insert(self):
test_shunt.setup()
db = asyncmongo.Client(pool_id='dup_insert',
host='127.0.0.1',
port=27017,
dbname='test')
def insert_callback(response, error):
tornado.ioloop.IOLoop.instance().stop()
logging.info(response)
assert len(response) == 1
test_shunt.register_called('inserted')
db.test_users.insert({"_id": "duplicate_insert.%d" % TEST_TIMESTAMP},
callback=insert_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('inserted')
def duplicate_callback(response, error):
tornado.ioloop.IOLoop.instance().stop()
logging.info(response)
if error:
test_shunt.register_called('dupe')
db.test_users.insert({"_id": "duplicate_insert.%d" % TEST_TIMESTAMP},
callback=duplicate_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('dupe')
def setup_db(cls):
cls.db = asyncmongo.Client(
pool_id='gamedb',
host='127.0.0.1',
port=27017,
dbname='tictactoe'
)
def test_query(self):
db = asyncmongo.Client(pool_id='test_query',
host='127.0.0.1',
port=int(self.mongod_options[0][1]),
dbname='test',
mincached=3)
def noop_callback(response, error):
logging.info(response)
loop = tornado.ioloop.IOLoop.instance()
# delay the stop so kill cursor has time on the ioloop to get pushed through to mongo
loop.add_timeout(time.time() + .1, loop.stop)
before = self.get_open_cursors()
# run 2 queries
db.foo.find({}, callback=noop_callback)
tornado.ioloop.IOLoop.instance().start()
db.foo.find({}, callback=noop_callback)
tornado.ioloop.IOLoop.instance().start()
# check cursors
after = self.get_open_cursors()
assert before == after, "%d cursors left open (should be 0)" % (after -
before)
def test_query(self):
logging.info('in test_query')
test_shunt.setup()
db = asyncmongo.Client(pool_id='test_query',
host='127.0.0.1',
port=27017,
dbname='test',
mincached=3)
def insert_callback(response, error):
logging.info(response)
assert len(response) == 1
test_shunt.register_called('inserted')
tornado.ioloop.IOLoop.instance().stop()
db.test_users.insert({"_id": "test_connection.%d" % TEST_TIMESTAMP},
safe=True,
callback=insert_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('inserted')
def callback(response, error):
assert len(response) == 1
test_shunt.register_called('got_record')
tornado.ioloop.IOLoop.instance().stop()
db.test_users.find({}, limit=1, callback=callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called("got_record")
def __init__(self):
handlers = [(r"/", WelcomeHandler), (r"/login", LoginHandler),
(r'/logout', LogoutHandler), (r'/signup', SignupHandler),
(r'/share', ShareHandler),
(r'/userexist', UserExistHandler),
(r'/qualifyCanvasCreation', qualifyCanvasCreationHandler),
(r'/createCanvas', createCanvasHandler)]
self.test3 = asyncmongo.Client(
pool_id="mydb",
host=os.environ['OPENSHIFT_MONGODB_DB_HOST'],
port=int(os.environ['OPENSHIFT_MONGODB_DB_PORT']),
dbname='test3',
dbuser=os.environ['OPENSHIFT_MONGODB_DB_USERNAME'],
dbpass=os.environ['OPENSHIFT_MONGODB_DB_PASSWORD'])
#self.test3 = asyncmongo.Client(pool_id="mydb",
# host="localhost",
# port=27017,
# dbname='test3')
settings = {
'debug':
True,
'template_path':
os.path.dirname(os.path.realpath(__file__)) + '/templates',
'static_path':
os.path.dirname(os.path.realpath(__file__)) + '/static',
'cookie_secret':
base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes),
'login_url':
'/login'
}
tornado.web.Application.__init__(self, handlers, **settings)
def test_authentication(self):
try:
test_shunt.setup()
db = asyncmongo.Client(pool_id='testauth', host='127.0.0.1', port=27018, dbname='test', dbuser='******',
dbpass='******', maxconnections=2)
def update_callback(response, error):
logging.info("UPDATE:")
tornado.ioloop.IOLoop.instance().stop()
logging.info(response)
assert len(response) == 1
test_shunt.register_called('update')
db.test_stats.update({"_id" : TEST_TIMESTAMP}, {'$inc' : {'test_count' : 1}}, upsert=True,
callback=update_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('update')
def query_callback(response, error):
tornado.ioloop.IOLoop.instance().stop()
logging.info(response)
logging.info(error)
assert error is None
assert isinstance(response, dict)
assert response['_id'] == TEST_TIMESTAMP
assert response['test_count'] == 1
test_shunt.register_called('retrieved')
db.test_stats.find_one({"_id" : TEST_TIMESTAMP}, callback=query_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('retrieved')
except:
tornado.ioloop.IOLoop.instance().stop()
raise
def _setup_connection(self):
self._db = asyncmongo.Client(pool_id='mydb',
maxcached=10,
maxconnections=50,
host=self.host,
port=self.port,
dbname=self.db_name)
def test_insert(self):
test_shunt.setup()
db = asyncmongo.Client(pool_id='testinsert', host='127.0.0.1', port=27017, dbname='test')
def insert_callback(response, error):
tornado.ioloop.IOLoop.instance().stop()
logging.info(response)
assert len(response) == 1
test_shunt.register_called('inserted')
db.test_users.insert({"_id" : "insert.%d" % TEST_TIMESTAMP}, callback=insert_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('inserted')
def query_callback(response, error):
tornado.ioloop.IOLoop.instance().stop()
logging.info(response)
assert len(response) == 1
test_shunt.register_called('retrieved')
db.test_users.find_one({"_id" : "insert.%d" % TEST_TIMESTAMP}, callback=query_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('retrieved')
def delete_callback(response, error):
tornado.ioloop.IOLoop.instance().stop()
logging.info(response)
assert len(response) == 1
test_shunt.register_called('deleted')
db.test_users.remove({"_id" : "insert.%d" % TEST_TIMESTAMP}, callback=delete_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('deleted')
def mongodb(self):
if not hasattr(self, '_mongodb'):
self._mongodb = asyncmongo.Client(
pool_id='mongostat',
host=self.application.mongo_address,
port=self.application.mongo_port,
dbname=self.application.mongo_database)
return self._mongodb
def db(self):
if not hasattr(self, '_db'):
self._db = asyncmongo.Client(
pool_id='uglyweb',
host=self.application.settings['db_host'],
port=self.application.settings['db_port'],
dbname=self.application.settings['db_name'])
return self._db
def initialize(self, db):
self.db = asyncmongo.Client(pool_id=db["pool_id"],
dbname=db["dbname"],
host=db.get("host", "127.0.0.1"),
port=db.get("port", 27017),
maxcached=db.get("maxcached", 10),
maxconnections=db.get(
"maxconnections", 50))
def test_update_safe():
test_shunt.setup()
db = asyncmongo.Client(pool_id='testinsert',
host='127.0.0.1',
port=27017,
dbname='test',
maxconnections=2)
def update_callback(response, error):
logging.info(response)
assert len(response) == 1
test_shunt.register_called('update')
tornado.ioloop.IOLoop.instance().stop()
# all of these should be called, but only one should have a callback
# we also are checking that connections in the pool never increases >1 with max_connections=2
# this is because connections for safe=False calls get put back in the pool immediated
db.test_stats.update({"_id": TEST_TIMESTAMP}, {'$inc': {
'test_count': 1
}},
safe=False,
upsert=True)
db.test_stats.update({"_id": TEST_TIMESTAMP}, {'$inc': {
'test_count': 1
}},
safe=False,
upsert=True)
db.test_stats.update({"_id": TEST_TIMESTAMP}, {'$inc': {
'test_count': 1
}},
safe=False,
upsert=True)
db.test_stats.update({"_id": TEST_TIMESTAMP}, {'$inc': {
'test_count': 1
}},
safe=False,
upsert=True)
db.test_stats.update({"_id": TEST_TIMESTAMP}, {'$inc': {
'test_count': 1
}},
upsert=True,
callback=update_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('update')
def query_callback(response, error):
logging.info(response)
assert isinstance(response, dict)
assert response['_id'] == TEST_TIMESTAMP
assert response['test_count'] == 5
test_shunt.register_called('retrieved')
tornado.ioloop.IOLoop.instance().stop()
db.test_stats.find_one({"_id": TEST_TIMESTAMP}, callback=query_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('retrieved')
def db(self):
if not hasattr(self, '_db'):
self._db = asyncmongo.Client(dbname=MONGO_DB,
host=MONGO_HOST,
port=MONGO_PORT,
maxcached=10,
maxconnections=50,
pool_id='mypool')
return self._db
def test_getitem(self):
db = asyncmongo.Client(pool_id='test_query',
host='127.0.0.1',
port=27017,
dbname='test',
mincached=3)
self.assert_(
repr(db['foo']) == repr(db.foo),
"dict-style access of a collection should be same as property access"
)
def db(self):
if not hasattr(self, "_db"):
self._db = asyncmongo.Client(pool_id=self.dbname,
mincached=0,
maxcached=10,
maxconnections=50,
maxusage=10,
dbname=self.dbname,
host="127.0.0.1",
port=27017)
return self._db
def db(self):
if not hasattr(self, '_db'):
self._db = asyncmongo.Client(
pool_id='mypool',
host='localhost',
port=27017,
maxcached=10,
maxconnections=50,
dbname='apps',
)
return self._db
def __init__(self, **kwargs):
self._conn = asyncmongo.Client(
pool_id=kwargs.get('pool_id', 'xcat.cache.Mongod'),
host=kwargs.get('host', '127.0.0.1'),
port=kwargs.get('port', 27017),
maxcached=kwargs.get('maxcached', 10),
maxconnections=kwargs.get('maxconnections', 50),
dbname=kwargs.get('dbname', 'cache'),
dbuser=kwargs.get('dbuser', None),
dbpass=kwargs.get('dbpass', None))
self._table = kwargs.get('table', 'caches')
def test_connection(self):
db = asyncmongo.Client(pool_id='test_query',
host='127.0.0.1',
port=27017,
dbname='test',
mincached=3)
for connection_name in [
'.', '..', '.foo', 'foo.', '.foo.', 'foo\x00'
'\x00foo'
]:
self.assertRaises(DataError,
lambda: db.connection(connection_name))
def test_find_and_modify(self):
db = asyncmongo.Client(pool_id='test_query',
host='127.0.0.1',
port=int(self.mongod_options[0][1]),
dbname='test',
mincached=3)
results = []
def callback(response, error):
tornado.ioloop.IOLoop.instance().stop()
self.assert_(error is None)
results.append(response['value'])
before = self.get_open_cursors()
# First findAndModify creates doc with i: 2 and s: 'a'
db.command(
'findAndModify',
'foo',
callback=callback,
query={'_id': 2},
update={'$set': {
's': 'a'
}},
upsert=True,
new=True,
)
tornado.ioloop.IOLoop.instance().start()
self.assertEqual({'_id': 2, 's': 'a'}, results[0])
# Second findAndModify updates doc with i: 2, sets s to 'b'
db.command(
'findAndModify',
'foo',
callback=callback,
query={'_id': 2},
update={'$set': {
's': 'b'
}},
upsert=True,
new=True,
)
tornado.ioloop.IOLoop.instance().start()
self.assertEqual({'_id': 2, 's': 'b'}, results[1])
# check cursors
after = self.get_open_cursors()
assert before == after, "%d cursors left open (should be 0)" % (after -
before)
def test_pooled_db(self):
"""
This tests simply verifies that we can grab two different connections from the pool
and use them independantly.
"""
print asyncmongo.__file__
test_shunt.setup()
client = asyncmongo.Client('id1',
maxconnections=5,
host='127.0.0.1',
port=27017,
dbname='test')
test_users_collection = client.connection('test_users')
def insert_callback(response, error):
logging.info(response)
assert len(response) == 1
test_shunt.register_called('inserted')
tornado.ioloop.IOLoop.instance().stop()
test_users_collection.insert(
{"_id": "record_test.%d" % TEST_TIMESTAMP},
safe=True,
callback=insert_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('inserted')
def pool_callback(response, error):
assert len(response) == 1
test_shunt.register_called('pool1')
if test_shunt.is_called('pool2'):
tornado.ioloop.IOLoop.instance().stop()
def pool_callback2(response, error):
assert len(response) == 1
test_shunt.register_called('pool2')
if test_shunt.is_called('pool1'):
# don't expect 2 finishes second
tornado.ioloop.IOLoop.instance().stop()
test_users_collection.find({}, limit=1, callback=pool_callback)
test_users_collection.find({}, limit=1, callback=pool_callback2)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('pool1')
test_shunt.assert_called('pool2')
def get_storage(self):
kwargs = self.settings
conn = asyncmongo.Client(pool_id=kwargs.get('pool_id',
'xcat.session.Mongod'),
host=kwargs.get('host', '127.0.0.1'),
port=kwargs.get('port', 27017),
maxcached=kwargs.get('maxcached', 10),
maxconnections=kwargs.get(
'maxconnections', 50),
dbname=kwargs.get('dbname', 'session'),
dbuser=kwargs.get('dbuser', None),
dbpass=kwargs.get('dbpass', None))
table = kwargs.get('table', 'sessions')
return self.Storage(conn, table, self.session_id, self.left_time)
def __init__(self):
self.db = asyncmongo.Client(pool_id='mydb',
host='127.0.0.1',
port=27017,
dbname='test')
handlers = [(r'/', MainHandler), (r'/count', CountHandler),
(r'/add_message', AddHandler),
(r'/chatsocket', ChatSocketHandler)]
settings = dict(
cookie_secret='43oETzKXQAGaYdk6fd8fG1kJFuYh7EQnp2XdTP1o/Vo=',
template_path=os.path.join(os.path.dirname(__file__), 'templates'),
static_path=os.path.join(os.path.dirname(__file__), 'static'),
xsrf_cookies=True,
autoescape=None
#debug=True
)
tornado.web.Application.__init__(self, handlers, **settings)
def too_many_connections(self):
clients = [
asyncmongo.Client('id2',
maxconnections=2,
host='127.0.0.1',
port=27017,
dbname='test') for i in range(3)
]
def callback(response, error):
pass
for client in clients[:2]:
client.connection('foo').find({}, callback=callback)
self.assertRaises(
TooManyConnections,
lambda: clients[2].connection('foo').find({}, callback=callback))
def test_stuff(self):
db = asyncmongo.Client(
pool_id='test_query',
host='127.0.0.1',
port=27017,
dbname='test',
mincached=3
)
yield gen.Task(db.collection.remove, safe=True)
yield gen.Task(db.collection.insert, {"_id" : 1}, safe=True)
# Verify the document was inserted
yield AssertEqual([{'_id': 1}], db.collection.find)
# MongoDB has a unique index on _id
yield AssertRaises(
asyncmongo.errors.IntegrityError,
db.collection.insert, {"_id" : 1}, safe=True)
def main():
global http_server
try:
signal.signal(signal.SIGTERM, on_signal)
tornado.options.parse_command_line()
if options.config != None:
tornado.options.parse_config_file(options.config)
path = os.path.join(os.path.dirname(__file__), "templates")
application = tornado.web.Application(
[(r'/', OverviewHandler),
(r'/dynamic', DynamicOverviewHandler, dict(template_path=path)),
(r'/query', MongoHandler)],
template_path=path,
static_path=os.path.join(os.path.dirname(__file__), "static"),
debug=True)
application.db = asyncmongo.Client(pool_id=str(uuid.uuid1()),
host=options.db_host,
port=options.db_port,
dbname=options.db_name)
http_server = tornado.httpserver.HTTPServer(application)
http_server.listen(options.port, options.address)
logging.getLogger().info("server listening on port %s:%d" %
(options.address, options.port))
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.getLogger().debug("autoreload enabled")
tornado.autoreload.start()
tornado.ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
logging.getLogger().info("exiting...")
except BaseException as ex:
logging.getLogger().error(
"exiting due: [%s][%s]" %
(str(ex), str(traceback.format_exc().splitlines())))
sys.exit(1)
def test_failed_auth(self):
try:
test_shunt.setup()
db = asyncmongo.Client(pool_id='testauth_f', host='127.0.0.1', port=27018, dbname='test', dbuser='******',
dbpass='******', maxconnections=2)
def query_callback(response, error):
tornado.ioloop.IOLoop.instance().stop()
logging.info(response)
logging.info(error)
assert isinstance(error, asyncmongo.AuthenticationError)
assert response is None
test_shunt.register_called('auth_failed')
db.test_stats.find_one({"_id" : TEST_TIMESTAMP}, callback=query_callback)
tornado.ioloop.IOLoop.instance().start()
test_shunt.assert_called('auth_failed')
except:
tornado.ioloop.IOLoop.instance().stop()
raise
def __init__(self):
self.__win = Gtk.Window()
self.__win.set_title("AsyncMongo test")
box = Gtk.VBox()
self.__win.add(box)
self.message = Gtk.Label('')
box.pack_start(self.message, 0, 1, 1)
btn = Gtk.Button(label="Test Insert")
box.pack_start(btn, 0, 1, 1)
btn.connect('clicked', self._on_insert_clicked)
btn = Gtk.Button(label="Test Query")
box.pack_start(btn, 0, 1, 1)
btn.connect('clicked', self._on_query_clicked)
self._db = asyncmongo.Client(pool_id='test_pool',
backend="glib3",
**database)
def __init__(self):
handlers = [
(r"/api/v1/article", ArticleHandler),
(r"/api/v1/is_loaded", IsLoadedHandler),
]
self.db = asyncmongo.Client(
pool_id='articles',
host=options.mongodb_host,
port=options.mongodb_port,
maxcached=options.mongodb_max_cached,
maxconnections=options.mongodb_max_connections,
dbname=options.mongodb_database)
self.doi_prefix = {}
if options.doi_prefix:
with open(options.doi_prefix) as f:
for line in f:
prefix = line.split("|")
self.doi_prefix[prefix[0]] = prefix[1]
self.article_types = {}
self.article_types['oa'] = 'research-article'
self.article_types['ab'] = 'abstract'
self.article_types['an'] = 'announcement'
self.article_types['co'] = 'article-commentary'
self.article_types['cr'] = 'case-report'
self.article_types['ed'] = 'editorial'
self.article_types['le'] = 'letter'
self.article_types['ra'] = 'review-article'
self.article_types['sc'] = 'rapid-communication'
self.article_types['nd'] = 'undefined'
#self.article_types['??'] = 'addendum'
#self.article_types['??'] = 'book-review'
#self.article_types['??'] = 'books-received'
#self.article_types['??'] = 'brief-report'
#self.article_types['??'] = 'calendar'
#self.article_types['??'] = 'collection'
#self.article_types['??'] = 'correction'
#self.article_types['??'] = 'discussion'
#self.article_types['??'] = 'dissertation'
#self.article_types['??'] = 'in-brief'
#self.article_types['??'] = 'introduction'
#self.article_types['??'] = 'meeting-report'
#self.article_types['??'] = 'news'
#self.article_types['??'] = 'obituary'
#self.article_types['??'] = 'oration'
#self.article_types['??'] = 'partial-retraction'
#self.article_types['??'] = 'product-review'
#self.article_types['??'] = 'reply'
#self.article_types['??'] = 'reprint'
#self.article_types['??'] = 'retraction'
#self.article_types['??'] = 'translation'
self.journal_subjects = {}
f_journal_subjects = open('_journal_subjects.txt', 'r')
s = f_journal_subjects.readlines()
for line in s:
parts = line.replace('\n', '').split('|')
if len(parts) == 2:
self.journal_subjects[parts[0]] = parts[1].split('#')
f_journal_subjects.close()
self.scielonet = {}
f_scielonet = open('_red.txt', 'r')
s = f_scielonet.readlines()
for line in s:
parts = line.replace('\n', '').split('|')
if len(parts) == 3:
if len(parts[1]) > 0:
self.scielonet[parts[0]] = parts[1]
f_scielonet.close()
settings = dict(template_path=os.path.join(os.path.dirname(__file__),
"templates"), )
# Local is the default the default way that ratchet works.
tornado.web.Application.__init__(self, handlers, **settings)
'''
Created on 2014年8月16日
@author: prstcsnpr
'''
import sys
import asyncmongo
import tornado
lines = []
db = asyncmongo.Client(pool_id="magicformula",
dbname="magicformula",
host="127.0.0.1",
port=27017)
def db_callback(response, error):
if len(lines) > 0:
fields = lines.pop()
db.stock_model.update({"ticker" : fields[2]},
{"$set" : {"category" : fields[1], "subcategory" : fields[0]}},
safe=True,
callback=db_callback)
else:
tornado.ioloop.IOLoop.instance().stop()
def main(argv):
