def test_unix_domain_socket(self): try: connection = Connection("/tmp/mongodb-27017.sock") connection.database_names() except: # Possibly the OS doesn't support unix domain sockets raise SkipTest("No Unix domain sockets")
def test_unix_domain_socket(self): try: connection = Connection("/tmp/mongodb-27017.sock") connection.database_names() except: # Possibly the OS doesn't support unix domain sockets raise SkipTest("No Unix domain sockets")
def test_ipv6(self): self.assertRaises(InvalidURI, _parse_uri, "::1", 27017) self.assertRaises(InvalidURI, _parse_uri, "[::1", 27017) self.assertRaises(InvalidURI, _parse_uri, "::1]:27017") self.assertRaises(InvalidURI, _parse_uri, "mongodb://::1", 27017) self.assert_(_parse_uri, "mongodb://[::1]:27017/?slaveOk=true") self.assert_(_parse_uri, "[::1]:27017,[2001:0db8:85a3:0000:0000:8a2e:0370:7334]" ":27018,192.168.0.212:27019,localhost:27020") self.assert_(_parse_uri, "mongodb://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]" ":27017/?slaveOk=true") try: connection = Connection("[::1]") except: # Either mongod was started without --ipv6 # or the OS doesn't support it (or both). raise SkipTest() # Try a few simple things connection = Connection("mongodb://[::1]:27017") connection = Connection("mongodb://[::1]:27017/?slaveOk=true") connection = Connection("[::1]:27017,localhost:27017") connection = Connection("localhost:27017,[::1]:27017") connection.pymongo_test.test.save({"dummy": u"object"}) connection.pymongo_test_bernie.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assert_("pymongo_test" in dbs) self.assert_("pymongo_test_bernie" in dbs)
def test_ipv6(self): self.assertRaises(InvalidURI, _parse_uri, "::1", 27017) self.assertRaises(InvalidURI, _parse_uri, "[::1", 27017) self.assertRaises(InvalidURI, _parse_uri, "::1]:27017") self.assertRaises(InvalidURI, _parse_uri, "mongodb://::1", 27017) self.assert_(_parse_uri, "mongodb://[::1]:27017/?slaveOk=true") self.assert_(_parse_uri, "[::1]:27017,[2001:0db8:85a3:0000:0000:8a2e:0370:7334]" ":27018,192.168.0.212:27019,localhost:27020") self.assert_(_parse_uri, "mongodb://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]" ":27017/?slaveOk=true") self.assertRaises(AutoReconnect, Connection, 'foo') try: connection = Connection("[::1]") except: # Either mongod was started without --ipv6 # or the OS doesn't support it (or both). raise SkipTest() # Try a few simple things connection = Connection("mongodb://[::1]:27017") connection = Connection("mongodb://[::1]:27017/?slaveOk=true") connection = Connection("[::1]:27017,localhost:27017") connection = Connection("localhost:27017,[::1]:27017") connection.pymongo_test.test.save({"dummy": u"object"}) connection.pymongo_test_bernie.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assert_("pymongo_test" in dbs) self.assert_("pymongo_test_bernie" in dbs)
def test_database_names(self): connection = Connection(self.host, self.port) connection.pymongo_test.test.save({"dummy": u"object"}) connection.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_mike" in dbs)
def test_database_names(self): connection = Connection(self.host, self.port) connection.pymongo_test.test.save({"dummy": u"object"}) connection.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assert_("pymongo_test" in dbs) self.assert_("pymongo_test_mike" in dbs)
def test_drop_database(self): connection = Connection(self.host, self.port) self.assertRaises(TypeError, connection.drop_database, 5) self.assertRaises(TypeError, connection.drop_database, None) connection.pymongo_test.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assert_("pymongo_test" in dbs) connection.drop_database("pymongo_test") dbs = connection.database_names() self.assert_("pymongo_test" not in dbs) connection.pymongo_test.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assert_("pymongo_test" in dbs) connection.drop_database(connection.pymongo_test) dbs = connection.database_names() self.assert_("pymongo_test" not in dbs)
def test_drop_database(self): connection = Connection(self.host, self.port) self.assertRaises(TypeError, connection.drop_database, 5) self.assertRaises(TypeError, connection.drop_database, None) connection.pymongo_test.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assert_("pymongo_test" in dbs) connection.drop_database("pymongo_test") dbs = connection.database_names() self.assert_("pymongo_test" not in dbs) connection.pymongo_test.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assert_("pymongo_test" in dbs) connection.drop_database(connection.pymongo_test) dbs = connection.database_names() self.assert_("pymongo_test" not in dbs)
def test_drop_database(self): connection = Connection(self.host, self.port) self.assertRaises(TypeError, connection.drop_database, 5) self.assertRaises(TypeError, connection.drop_database, None) raise SkipTest("This test often fails due to SERVER-2329") connection.pymongo_test.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assertTrue("pymongo_test" in dbs) connection.drop_database("pymongo_test") dbs = connection.database_names() self.assertTrue("pymongo_test" not in dbs) connection.pymongo_test.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assertTrue("pymongo_test" in dbs) connection.drop_database(connection.pymongo_test) dbs = connection.database_names() self.assertTrue("pymongo_test" not in dbs)
def test_drop_database(self): connection = Connection(self.host, self.port) self.assertRaises(TypeError, connection.drop_database, 5) self.assertRaises(TypeError, connection.drop_database, None) raise SkipTest("This test often fails due to SERVER-2329") connection.pymongo_test.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assertTrue("pymongo_test" in dbs) connection.drop_database("pymongo_test") dbs = connection.database_names() self.assertTrue("pymongo_test" not in dbs) connection.pymongo_test.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assertTrue("pymongo_test" in dbs) connection.drop_database(connection.pymongo_test) dbs = connection.database_names() self.assertTrue("pymongo_test" not in dbs)
def test_copy_db(self): c = Connection(self.host, self.port) self.assertTrue(c.in_request()) self.assertRaises(TypeError, c.copy_database, 4, "foo") self.assertRaises(TypeError, c.copy_database, "foo", 4) self.assertRaises(InvalidName, c.copy_database, "foo", "$foo") c.pymongo_test.test.drop() c.drop_database("pymongo_test1") c.drop_database("pymongo_test2") c.pymongo_test.test.insert({"foo": "bar"}) # Due to SERVER-2329, databases may not disappear from a master in a # master-slave pair if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) self.assertFalse("pymongo_test2" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1") # copy_database() didn't accidentally end the request self.assertTrue(c.in_request()) self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) c.end_request() self.assertFalse(c.in_request()) c.copy_database("pymongo_test", "pymongo_test2", "%s:%d" % (self.host, self.port)) # copy_database() didn't accidentally restart the request self.assertFalse(c.in_request()) self.assertTrue("pymongo_test2" in c.database_names()) self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"]) if version.at_least(c, (1, 3, 3, 1)): c.drop_database("pymongo_test1") c.pymongo_test.add_user("mike", "password") self.assertRaises( OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******" ) if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) self.assertRaises( OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******" ) if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) if not is_mongos(c): # See SERVER-6427 c.copy_database("pymongo_test", "pymongo_test1", username="******", password="******") self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
def test_copy_db(self): c = Connection(self.host, self.port) self.assertRaises(TypeError, c.copy_database, 4, "foo") self.assertRaises(TypeError, c.copy_database, "foo", 4) self.assertRaises(InvalidName, c.copy_database, "foo", "$foo") c.pymongo_test.test.drop() c.drop_database("pymongo_test1") c.drop_database("pymongo_test2") c.pymongo_test.test.insert({"foo": "bar"}) self.assertFalse("pymongo_test1" in c.database_names()) self.assertFalse("pymongo_test2" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1") self.assert_("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) c.copy_database("pymongo_test", "pymongo_test2", "%s:%s" % (self.host, self.port)) self.assert_("pymongo_test2" in c.database_names()) self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"]) if version.at_least(c, (1, 3, 3, 1)): c.drop_database("pymongo_test1") c.pymongo_test.add_user("mike", "password") self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.assertFalse("pymongo_test1" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1", username="******", password="******") self.assert_("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])
def test_ipv6(self): try: connection = Connection("[::1]") except: # Either mongod was started without --ipv6 # or the OS doesn't support it (or both). raise SkipTest() # Try a few simple things connection = Connection("mongodb://[::1]:27017") connection = Connection("mongodb://[::1]:27017/?slaveOk=true") connection = Connection("[::1]:27017,localhost:27017") connection = Connection("localhost:27017,[::1]:27017") connection.pymongo_test.test.save({"dummy": u"object"}) connection.pymongo_test_bernie.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assert_("pymongo_test" in dbs) self.assert_("pymongo_test_bernie" in dbs)
def test_ipv6(self): try: connection = Connection("[::1]") except: # Either mongod was started without --ipv6 # or the OS doesn't support it (or both). raise SkipTest() # Try a few simple things connection = Connection("mongodb://[::1]:27017") connection = Connection("mongodb://[::1]:27017/?slaveOk=true") connection = Connection("[::1]:27017,localhost:27017") connection = Connection("localhost:27017,[::1]:27017") connection.pymongo_test.test.save({"dummy": u"object"}) connection.pymongo_test_bernie.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assert_("pymongo_test" in dbs) self.assert_("pymongo_test_bernie" in dbs)
def test_unix_socket(self): if not hasattr(socket, "AF_UNIX"): raise SkipTest("UNIX-sockets are not supported on this system") mongodb_socket = "/tmp/mongodb-27017.sock" if not os.access(mongodb_socket, os.R_OK): raise SkipTest("Socket file is not accessable") self.assertTrue(Connection("mongodb://%s" % mongodb_socket)) connection = Connection("mongodb://%s" % mongodb_socket) connection.pymongo_test.test.save({"dummy": "object"}) # Confirm we can read via the socket dbs = connection.database_names() self.assertTrue("pymongo_test" in dbs) # Confirm it fails with a missing socket self.assertRaises(ConnectionFailure, Connection, "mongodb:///tmp/none-existent.sock")
def test_copy_db(self): c = Connection(self.host, self.port) self.assertRaises(TypeError, c.copy_database, 4, "foo") self.assertRaises(TypeError, c.copy_database, "foo", 4) self.assertRaises(InvalidName, c.copy_database, "foo", "$foo") c.drop_database("pymongo_test") c.drop_database("pymongo_test1") c.drop_database("pymongo_test2") c.pymongo_test.test.insert({"foo": "bar"}) self.failIf("pymongo_test1" in c.database_names()) self.failIf("pymongo_test2" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1") self.assert_("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) c.copy_database("pymongo_test", "pymongo_test2", "%s:%s" % (self.host, self.port)) self.assert_("pymongo_test2" in c.database_names()) self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"]) c.drop_database("pymongo_test1") c.drop_database("pymongo_test2") if version.at_least(c, (1, 3, 3, 1)): c.pymongo_test.add_user("mike", "password") self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.failIf("pymongo_test1" in c.database_names()) self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") self.failIf("pymongo_test1" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1", username="******", password="******") self.assert_("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) c.drop_database("pymongo_test1")
def test_unix_socket(self): if not hasattr(socket, "AF_UNIX"): raise SkipTest("UNIX-sockets are not supported on this system") mongodb_socket = '/tmp/mongodb-27017.sock' if not os.access(mongodb_socket, os.R_OK): raise SkipTest("Socket file is not accessable") self.assertTrue(Connection("mongodb://%s" % mongodb_socket)) connection = Connection("mongodb://%s" % mongodb_socket) connection.pymongo_test.test.save({"dummy": "object"}) # Confirm we can read via the socket dbs = connection.database_names() self.assertTrue("pymongo_test" in dbs) # Confirm it fails with a missing socket self.assertRaises(ConnectionFailure, Connection, "mongodb:///tmp/none-existent.sock")
def test_unix_socket(self): if not hasattr(socket, "AF_UNIX"): raise SkipTest("UNIX-sockets are not supported on this system") if (sys.platform == 'darwin' and server_started_with_auth(Connection(self.host, self.port))): raise SkipTest("SERVER-8492") mongodb_socket = '/tmp/mongodb-27017.sock' if not os.access(mongodb_socket, os.R_OK): raise SkipTest("Socket file is not accessable") self.assertTrue(Connection("mongodb://%s" % mongodb_socket)) connection = Connection("mongodb://%s" % mongodb_socket) connection.pymongo_test.test.save({"dummy": "object"}, safe=True) # Confirm we can read via the socket dbs = connection.database_names() self.assertTrue("pymongo_test" in dbs) # Confirm it fails with a missing socket self.assertRaises(ConnectionFailure, Connection, "mongodb:///tmp/none-existent.sock")
class DASCacheModel(DASWebManager): """ DASCacheModel represents DAS cache RESTful interface. It supports POST/GET/DELETE/UPDATE methods who communicate with DAS caching systems. The input queries are placed into DAS cache queue and served via FIFO mechanism. """ def __init__(self, config): self.config = config DASWebManager.__init__(self, config) self.version = __version__ self.methods = {} self.methods['GET'] = { 'request': { 'args': ['idx', 'limit', 'query', 'skey', 'order'], 'call': self.request, 'version': __version__ }, 'nresults': { 'args': ['query'], 'call': self.nresults, 'version': __version__ }, 'records': { 'args': ['query', 'count', 'collection'], 'call': self.records, 'version': __version__ }, 'status': { 'args': ['query'], 'call': self.status, 'version': __version__ }, } self.methods['POST'] = { 'create': { 'args': ['query', 'expire'], 'call': self.create, 'version': __version__ } } self.methods['PUT'] = { 'replace': { 'args': ['query', 'expire'], 'call': self.replace, 'version': __version__ } } self.methods['DELETE'] = { 'delete': { 'args': ['query'], 'call': self.delete, 'version': __version__ } } try: # WMCore/WebTools rest = RESTModel(config) rest.methods = self.methods # set RESTModel methods self.model = self # re-reference model to my class self.model.handler = rest.handler # reference handler to RESTModel cdict = self.config.dictionary_() self.base = '/rest' except: cdict = {} self.base = '' self.dascore = DASCore() dbhost = self.dascore.dasconfig['mongocache_dbhost'] dbport = self.dascore.dasconfig['mongocache_dbport'] capped_size = self.dascore.dasconfig['mongocache_capped_size'] self.con = Connection(dbhost, dbport) if 'logging' not in self.con.database_names(): db = self.con['logging'] options = {'capped': True, 'size': capped_size} db.create_collection('db', options) self.warning('Created logging.db, size=%s' % capped_size) self.col = self.con['logging']['db'] sleep = cdict.get('sleep', 2) verbose = cdict.get('verbose', None) iconfig = { 'sleep': sleep, 'verbose': verbose, 'logger': self.dascore.logger } self.cachemgr = DASCacheMgr(iconfig) thread.start_new_thread(self.cachemgr.worker, (worker, )) msg = 'DASCacheMode::init, host=%s, port=%s, capped_size=%s' \ % (dbhost, dbport, capped_size) self.dascore.logger.debug(msg) print(msg) def logdb(self, query): """ Make entry in Logging DB """ qhash = genkey(query) headers = cherrypy.request.headers doc = dict(qhash=qhash, timestamp=time.time(), headers=cherrypy.request.headers, method=cherrypy.request.method, path=cherrypy.request.path_info, args=cherrypy.request.params, ip=cherrypy.request.remote.ip, hostname=cherrypy.request.remote.name, port=cherrypy.request.remote.port) self.col.insert(doc) @checkargs def records(self, *args, **kwargs): """ HTTP GET request. Retrieve records from provided collection. """ data = {'server_method': 'request'} if 'query' not in kwargs: data['status'] = 'fail' data['reason'] = 'no query is provided' return data # input query in JSON format, we should decode it using json. query = json.loads(kwargs.get('query')) coll = kwargs.get('collection', 'merge') idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 10) # getarg perfrom type convertion count = kwargs.get('count', 0) data.update({ 'status': 'requested', 'query': kwargs['query'], 'collection': coll, 'count': count }) if '_id' in query['spec']: recid = query['spec']['_id'] ids = [] if type(recid) is bytes: ids = [ObjectId(recid)] elif type(recid) is list: ids = [ObjectId(r) for r in recid] spec = {'spec': {'_id': {'$in': ids}}} else: # look-up all records spec = {} self.logdb(query) try: gen = self.dascore.rawcache.get_from_cache\ (spec, idx=idx, limit=limit, collection=coll, adjust=False) data['status'] = 'success' data['data'] = [r for r in gen] except: self.debug(traceback.format_exc()) data['status'] = 'fail' data['reason'] = sys.exc_info()[0] return data @checkargs def status(self, *args, **kwargs): """ HTTP GET request. Check status of the input query in DAS. """ data = {'server_method': 'status'} if 'query' in kwargs: query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) status = self.dascore.get_status(query) if not status: status = 'no data' data.update({'status': status}) else: data.update({ 'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @checkargs def nresults(self, *args, **kwargs): """ HTTP GET request. Ask DAS for total number of records for provided query. """ data = {'server_method': 'nresults'} if 'query' in kwargs: query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) data.update({'status': 'success'}) res = self.dascore.in_raw_cache_nresults(query) data.update({'status': 'success', 'nresults': res}) else: data.update({ 'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @checkargs def request(self, *args, **kwargs): """ HTTP GET request. Retrieve results from DAS cache. """ data = {'server_method': 'request'} if 'query' in kwargs: query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 0) skey = getarg(kwargs, 'skey', '') order = getarg(kwargs, 'order', 'asc') data.update({ 'status': 'requested', 'idx': idx, 'limit': limit, 'query': query, 'skey': skey, 'order': order }) # if self.dascore.in_raw_cache(query): res = self.dascore.result(query, idx, limit) if type(res) is types.GeneratorType: result = [] for item in res: if item not in result: result.append(item) data['data'] = result tot = len(data['data']) else: data['data'] = res tot = 1 data['status'] = 'success' data['nresults'] = tot # else: # data['status'] = 'not found' else: data.update({ 'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @checkargs def create(self, *args, **kwargs): """ HTTP POST request. Requests the server to create a new resource using the data enclosed in the request body. Creates new entry in DAS cache for provided query. """ data = {'server_method': 'create'} if 'query' in kwargs: query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) expire = getarg(kwargs, 'expire', 600) try: status = self.cachemgr.add(query, expire) data.update({ 'status': status, 'query': query, 'expire': expire }) except: data.update({ 'exception': traceback.format_exc(), 'status': 'fail' }) else: data.update({ 'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @checkargs def replace(self, *args, **kwargs): """ HTTP PUT request. Requests the server to replace an existing resource with the one enclosed in the request body. Replace existing query in DAS cache. """ data = {'server_method': 'replace'} if 'query' in kwargs: query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) try: self.dascore.remove_from_cache(query) except: msg = traceback.format_exc() data.update({ 'status': 'fail', 'query': query, 'exception': msg }) return data expire = getarg(kwargs, 'expire', 600) try: status = self.cachemgr.add(query, expire) data.update({ 'status': status, 'query': query, 'expire': expire }) except: data.update({ 'status': 'fail', 'query': query, 'exception': traceback.format_exc() }) else: data.update({ 'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @checkargs def delete(self, *args, **kwargs): """ HTTP DELETE request. Delete input query in DAS cache """ data = {'server_method': 'delete'} if 'query' in kwargs: query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) data.update({'status': 'requested', 'query': query}) try: self.dascore.remove_from_cache(query) data.update({'status': 'success'}) except: msg = traceback.format_exc() data.update({'status': 'fail', 'exception': msg}) else: data.update({ 'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @exposejson def rest(self, *args, **kwargs): """ RESTful interface. We use args tuple as access method(s), e.g. args = ('method',) and kwargs to represent input parameters. """ request = cherrypy.request.method if request not in self.methods.keys(): msg = "Usupported request '%s'" % requset return {'error': msg} method = args[0] if method not in self.methods[request].keys(): msg = "Unsupported method '%s'" % method return {'error': msg} if request == 'POST': if cherrypy.request.body: body = cherrypy.request.body.read() try: kwargs = json.loads(body) except: msg = "Unable to load body request" return {'error': msg} return getattr(self, method)(kwargs)
class DASCacheModel(DASWebManager): """ DASCacheModel represents DAS cache RESTful interface. It supports POST/GET/DELETE/UPDATE methods who communicate with DAS caching systems. The input queries are placed into DAS cache queue and served via FIFO mechanism. """ def __init__(self, config): self.config = config DASWebManager.__init__(self, config) self.version = __version__ self.methods = {} self.methods['GET']= { 'request': {'args':['idx', 'limit', 'query', 'skey', 'order'], 'call': self.request, 'version':__version__}, 'nresults': {'args':['query'], 'call': self.nresults, 'version':__version__}, 'records': {'args':['query', 'count', 'collection'], 'call': self.records, 'version':__version__}, 'status': {'args':['query'], 'call': self.status, 'version':__version__}, } self.methods['POST']= {'create': {'args':['query', 'expire'], 'call': self.create, 'version':__version__}} self.methods['PUT']= {'replace': {'args':['query', 'expire'], 'call': self.replace, 'version':__version__}} self.methods['DELETE']= {'delete': {'args':['query'], 'call': self.delete, 'version':__version__}} try: # WMCore/WebTools rest = RESTModel(config) rest.methods = self.methods # set RESTModel methods self.model = self # re-reference model to my class self.model.handler = rest.handler # reference handler to RESTModel cdict = self.config.dictionary_() self.base = '/rest' except: cdict = {} self.base = '' self.dascore = DASCore() dbhost = self.dascore.dasconfig['mongocache_dbhost'] dbport = self.dascore.dasconfig['mongocache_dbport'] capped_size = self.dascore.dasconfig['mongocache_capped_size'] self.con = Connection(dbhost, dbport) if 'logging' not in self.con.database_names(): db = self.con['logging'] options = {'capped':True, 'size': capped_size} db.create_collection('db', options) self.warning('Created logging.db, size=%s' % capped_size) self.col = self.con['logging']['db'] sleep = cdict.get('sleep', 2) verbose = cdict.get('verbose', None) iconfig = {'sleep':sleep, 'verbose':verbose, 'logger':self.dascore.logger} self.cachemgr = DASCacheMgr(iconfig) thread.start_new_thread(self.cachemgr.worker, (worker, )) msg = 'DASCacheMode::init, host=%s, port=%s, capped_size=%s' \ % (dbhost, dbport, capped_size) self.dascore.logger.debug(msg) print msg def logdb(self, query): """ Make entry in Logging DB """ qhash = genkey(query) headers = cherrypy.request.headers doc = dict(qhash=qhash, timestamp=time.time(), headers=cherrypy.request.headers, method=cherrypy.request.method, path=cherrypy.request.path_info, args=cherrypy.request.params, ip=cherrypy.request.remote.ip, hostname=cherrypy.request.remote.name, port=cherrypy.request.remote.port) self.col.insert(doc) @checkargs def records(self, *args, **kwargs): """ HTTP GET request. Retrieve records from provided collection. """ data = {'server_method':'request'} if not kwargs.has_key('query'): data['status'] = 'fail' data['reason'] = 'no query is provided' return data # input query in JSON format, we should decode it using json. query = json.loads(kwargs.get('query')) coll = kwargs.get('collection', 'merge') idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 10) # getarg perfrom type convertion count = kwargs.get('count', 0) data.update({'status':'requested', 'query':kwargs['query'], 'collection':coll, 'count': count}) if query['spec'].has_key('_id'): recid = query['spec']['_id'] ids = [] if type(recid) is types.StringType: ids = [ObjectId(recid)] elif type(recid) is types.ListType: ids = [ObjectId(r) for r in recid] spec = {'spec':{'_id':{'$in':ids}}} else: # look-up all records spec = {} self.logdb(query) try: gen = self.dascore.rawcache.get_from_cache\ (spec, idx=idx, limit=limit, collection=coll, adjust=False) data['status'] = 'success' data['data'] = [r for r in gen] except: self.debug(traceback.format_exc()) data['status'] = 'fail' data['reason'] = sys.exc_type return data @checkargs def status(self, *args, **kwargs): """ HTTP GET request. Check status of the input query in DAS. """ data = {'server_method':'status'} if kwargs.has_key('query'): query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) status = self.dascore.get_status(query) if not status: status = 'no data' data.update({'status':status}) else: data.update({'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @checkargs def nresults(self, *args, **kwargs): """ HTTP GET request. Ask DAS for total number of records for provided query. """ data = {'server_method':'nresults'} if kwargs.has_key('query'): query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) data.update({'status':'success'}) res = self.dascore.in_raw_cache_nresults(query) data.update({'status':'success', 'nresults':res}) else: data.update({'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @checkargs def request(self, *args, **kwargs): """ HTTP GET request. Retrieve results from DAS cache. """ data = {'server_method':'request'} if kwargs.has_key('query'): query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 0) skey = getarg(kwargs, 'skey', '') order = getarg(kwargs, 'order', 'asc') data.update({'status':'requested', 'idx':idx, 'limit':limit, 'query':query, 'skey':skey, 'order':order}) # if self.dascore.in_raw_cache(query): res = self.dascore.result(query, idx, limit) if type(res) is types.GeneratorType: result = [] for item in res: if item not in result: result.append(item) data['data'] = result tot = len(data['data']) else: data['data'] = res tot = 1 data['status'] = 'success' data['nresults'] = tot # else: # data['status'] = 'not found' else: data.update({'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @checkargs def create(self, *args, **kwargs): """ HTTP POST request. Requests the server to create a new resource using the data enclosed in the request body. Creates new entry in DAS cache for provided query. """ data = {'server_method':'create'} if kwargs.has_key('query'): query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) expire = getarg(kwargs, 'expire', 600) try: status = self.cachemgr.add(query, expire) data.update({'status':status, 'query':query, 'expire':expire}) except: data.update({'exception':traceback.format_exc(), 'status':'fail'}) else: data.update({'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @checkargs def replace(self, *args, **kwargs): """ HTTP PUT request. Requests the server to replace an existing resource with the one enclosed in the request body. Replace existing query in DAS cache. """ data = {'server_method':'replace'} if kwargs.has_key('query'): query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) try: self.dascore.remove_from_cache(query) except: msg = traceback.format_exc() data.update({'status':'fail', 'query':query, 'exception':msg}) return data expire = getarg(kwargs, 'expire', 600) try: status = self.cachemgr.add(query, expire) data.update({'status':status, 'query':query, 'expire':expire}) except: data.update({'status':'fail', 'query':query, 'exception':traceback.format_exc()}) else: data.update({'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @checkargs def delete(self, *args, **kwargs): """ HTTP DELETE request. Delete input query in DAS cache """ data = {'server_method':'delete'} if kwargs.has_key('query'): query = kwargs['query'] self.logdb(query) query = self.dascore.mongoparser.parse(query) data.update({'status':'requested', 'query':query}) try: self.dascore.remove_from_cache(query) data.update({'status':'success'}) except: msg = traceback.format_exc() data.update({'status':'fail', 'exception':msg}) else: data.update({'status': 'fail', 'reason': 'Unsupported keys %s' % kwargs.keys() }) return data @exposejson def rest(self, *args, **kwargs): """ RESTful interface. We use args tuple as access method(s), e.g. args = ('method',) and kwargs to represent input parameters. """ request = cherrypy.request.method if request not in self.methods.keys(): msg = "Usupported request '%s'" % requset return {'error': msg} method = args[0] if method not in self.methods[request].keys(): msg = "Unsupported method '%s'" % method return {'error': msg} if request == 'POST': if cherrypy.request.body: body = cherrypy.request.body.read() try: kwargs = json.loads(body) except: msg = "Unable to load body request" return {'error': msg} return getattr(self, method)(kwargs)
def test_copy_db(self): c = Connection(self.host, self.port) self.assertTrue(c.in_request()) self.assertRaises(TypeError, c.copy_database, 4, "foo") self.assertRaises(TypeError, c.copy_database, "foo", 4) self.assertRaises(InvalidName, c.copy_database, "foo", "$foo") c.pymongo_test.test.drop() c.drop_database("pymongo_test1") c.drop_database("pymongo_test2") c.pymongo_test.test.insert({"foo": "bar"}) # Due to SERVER-2329, databases may not disappear from a master in a # master-slave pair if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) self.assertFalse("pymongo_test2" in c.database_names()) c.copy_database("pymongo_test", "pymongo_test1") # copy_database() didn't accidentally end the request self.assertTrue(c.in_request()) self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"]) c.end_request() c.copy_database("pymongo_test", "pymongo_test2", "%s:%d" % (self.host, self.port)) # copy_database() didn't accidentally restart the request self.assertFalse(c.in_request()) self.assertTrue("pymongo_test2" in c.database_names()) self.assertEqual("bar", c.pymongo_test2.test.find_one()["foo"]) if version.at_least(c, (1, 3, 3, 1)): c.drop_database("pymongo_test1") c.pymongo_test.add_user("mike", "password") self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) self.assertRaises(OperationFailure, c.copy_database, "pymongo_test", "pymongo_test1", username="******", password="******") if not server_is_master_with_slave(c): self.assertFalse("pymongo_test1" in c.database_names()) if not is_mongos(c): # See SERVER-6427 c.copy_database("pymongo_test", "pymongo_test1", username="******", password="******") self.assertTrue("pymongo_test1" in c.database_names()) self.assertEqual("bar", c.pymongo_test1.test.find_one()["foo"])