def setUp(self): self.settings = { "couchdb_uri": "http://*****:*****@cdb:5984", "couchdb_db": "desk_tester", } s = Server(self.settings["couchdb_uri"]) self.s = s s.create_db(self.settings['couchdb_db']) self.up = CouchdbUploader(path=os.path.dirname(__file__), auth=('admin', 'admin'), **self.settings)
class CouchDBServer(object): def __init__(self): self.__get_server_uri() self.__authenticate() self.__connect() def __get_server_uri(self): couchdb_port = config.couchdb.port if config.couchdb.protocol == 'http' else config.couchdb.ssl_port self.__couchdb_uri = "%s://%s:%s" % (config.couchdb.protocol, config.couchdb.host, couchdb_port) def __authenticate(self): user, passwd = config.couchdb.user, config.couchdb.password if all((user, passwd)): auth = restkit.BasicAuth(user, passwd) self.__auth_resource = CouchdbResource(filters=[auth]) else: self.__auth_resource = None def __connect(self): self.__server = Server(uri=self.__couchdb_uri, resource_instance=self.__auth_resource) def list_workspaces(self): return filter(is_usable_workspace, self.__server.all_dbs()) def get_workspace_handler(self, ws_name): return self.__server.get_db(ws_name) def get_or_create_db(self, ws_name): return self.__server.get_or_create_db(ws_name) def create_db(self, ws_name): return self.__server.create_db(ws_name) def delete_db(self, ws_name): return self.__server.delete_db(ws_name)
def setUp(self): self.db_conf = { "couchdb_uri": "http://*****:*****@cdb:5984", "couchdb_db": "desk_tester", } self.conf = { "powerdns_backend": "sqlite", "powerdns_db": "/var/services/data/powerdns/pdns_dnsa.test.sqlite3", "powerdns_name": "dnsa.test", "powerdns_primary": "dnsa.test", "worker_is_foreman": True, } self.conf.update(self.db_conf) s = Server(self.db_conf["couchdb_uri"]) self.s = s s.create_db(self.db_conf['couchdb_db']) self.db = self.s.get_db(self.db_conf["couchdb_db"]) self.couch_up = CouchdbUploader( path=os.path.dirname(__file__), auth=('admin', 'admin'), **self.db_conf ) status_code = self.couch_up.put( data="@fixtures/couchdb-design.json", doc_id="_design/{couchdb_db}" ) if not status_code == 201: s.delete_db(self.db_conf["couchdb_db"]) worker_id = "worker-foreman" d = { "_id": worker_id, "type": "worker", "hostname": "foreman", "provides": { "domain": [{"backend": "powerdns", "name": "dnsa.test"}] } } self.assertTrue( self.couch_up.put(data=json.dumps(d), doc_id=worker_id) == 201 ) self.assertTrue( self.couch_up.put(data="@fixtures/couchdb-template-dns.json", doc_id="template-email") == 201 ) self.assertTrue( self.couch_up.put(data="@fixtures/couchdb-map-ips.json", doc_id="map-ips") == 201 )
def create_db(): """create the db if it don't exist""" from couchdbkit import Server server = Server(settings.SERVER_URI) try: db = server.create_db(settings.DATABASE_NAME) except: pass
class Couch(): def __init__(self): self.server = Server() self.db = self.server.get_or_create_db('test') if self.db.info()['doc_count'] > 0: self.server.delete_db('test') self.db = self.server.create_db('test') def populate(self): things = [ {"name": "Vishnu"}, {"name": "Lakshmi"}, {"name": "Ganesha"}, {"name": "Krishna"}, {"name": "Murugan"} ] self.db.save_docs(things) def count(self): return self.db.all_docs().count()
class CouchDBServer(object): def __init__(self): self.__get_server_uri() self.__authenticate() self.__connect() def __get_server_uri(self): couchdb_port = config.couchdb.port if config.couchdb.protocol == 'http' else config.couchdb.ssl_port self.__couchdb_uri = "%s://%s:%s" % (config.couchdb.protocol, config.couchdb.host, couchdb_port) def __authenticate(self): user, passwd = config.couchdb.user, config.couchdb.password if all((user, passwd)): auth = restkit.BasicAuth(user, passwd) self.__auth_resource = CouchdbResource(filters=[auth]) else: self.__auth_resource = None def __connect(self): self.__server = Server(uri=self.__couchdb_uri, resource_instance=self.__auth_resource) def list_workspaces(self): return filter(is_usable_workspace, self.__server.all_dbs()) def get_workspace_handler(self, ws_name): return self.__server.get_db(ws_name) def get_or_create_db(self, ws_name): return self.__server.get_or_create_db(ws_name) def create_db(self, ws_name): return self.__server.create_db(ws_name) def delete_db(self, ws_name): return self.__server.delete_db(ws_name)
class Couch(): def __init__(self): self.server = Server() self.db = self.server.get_or_create_db('test') if self.db.info()['doc_count'] > 0: self.server.delete_db('test') self.db = self.server.create_db('test') def populate(self): things = [{ "name": "Vishnu" }, { "name": "Lakshmi" }, { "name": "Ganesha" }, { "name": "Krishna" }, { "name": "Murugan" }] self.db.save_docs(things) def count(self): return self.db.all_docs().count()
class DbConnectorCouchTestSuite(unittest.TestCase): def setUp(self): self.couch_srv = Server(uri=CONF.getCouchURI()) self.db_name = new_random_workspace_name() self.db = self.couch_srv.create_db(self.db_name) def tearDown(self): self.couch_srv.delete_db(self.db_name) time.sleep(3) def test_save_Document(self): couchConnector = CouchDbConnector(self.db) doc = {'_id': '123', 'data': 'some data'} couchConnector.saveDocument(doc) doc_from_db = self.db.get('123') self.assertNotEquals(doc_from_db, None, "Document should be retrieved") self.assertEquals(doc_from_db.get('data'), 'some data', "Data retrieved should be the same as data saved") def test_get_Document(self): couchConnector = CouchDbConnector(self.db) doc = {'_id': '123', 'data': 'some data'} couchConnector.saveDocument(doc) doc_retrieved = couchConnector.getDocument('123') self.assertNotEquals(doc_retrieved, None, "Document should be retrieved") self.assertEquals(doc_retrieved.get('data'), 'some data', "Data retrieved should be the same as data saved") def test_remove_Document(self): couchConnector = CouchDbConnector(self.db) doc = {'_id': '123', 'data': 'some data'} couchConnector.saveDocument(doc) couchConnector.remove('123') try: doc_from_db = self.db.get('123') except ResourceNotFound: doc_from_db = None self.assertEquals(doc_from_db, None, "Document should be None") def test_get_by_parent_and_type(self): couchConnector = CouchDbConnector(self.db) doc = { '_id': '123', 'type': 'father', 'parent': None, } couchConnector.saveDocument(doc) doc = { '_id': '456', 'type': 'child', 'parent': '123', } couchConnector.saveDocument(doc) doc = { '_id': '789', 'type': 'child', 'parent': '123', } couchConnector.saveDocument(doc) ids = couchConnector.getDocsByFilter(parentId='123', type='child') self.assertEquals(len(ids), 2, "There should be two 'childs' with parent '123'") self.assertIn('456', ids, "Child '456' should be in the list of childs") self.assertIn('789', ids, "Child '789' should be in the list of childs") ids = couchConnector.getDocsByFilter(parentId='123', type='son') self.assertEquals(len(ids), 0, "There shouldn't be any 'son' with parent '123'") ids = couchConnector.getDocsByFilter(parentId='456', type='child') self.assertEquals(len(ids), 0, "There shouldn't be any 'child' with parent '456'")
class CouchdbManager(PersistenceManager): """ This is a couchdb manager for the workspace, it will load from the couchdb databases""" def __init__(self, uri): self._last_seq_ack = 0 model.api.log("Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self.__dbs = {} self.__seq_nums = {} self.__serv = NoConectionServer() self.mutex = threading.Lock() self._available = False try: self.testCouchUrl(uri) url = urlparse(uri) print("Setting user,pass %s %s" % (url.username, url.password)) self.__serv = Server(uri=uri) #print dir(self.__serv) self.__serv.resource_class.credentials = (url.username, url.password) self._available = True except: model.api.log("No route to couchdb server on: %s" % uri) print(traceback.format_exc()) def isAvailable(self): return self._available def lostConnectionResolv(self): self._lostConnection = True self.__dbs.clear() self.__serv = NoConectionServer() def reconnect(self): ret_val = False ur = self.__uri if CouchdbManager.testCouch(ur): self.__serv = Server(uri=ur) self.__dbs.clear() self._lostConnection = False ret_val = True return ret_val @staticmethod def testCouch(uri): host, port = None, None try: import socket url = urlparse(uri) proto = url.scheme host = url.hostname port = url.port port = port if port else socket.getservbyname(proto) s = socket.socket() s.settimeout(1) s.connect((host, int(port))) except: return False model.api.log("Connecting Couch to: %s:%s" % (host, port)) return True def testCouchUrl(self, uri): url = urlparse(uri) proto = url.scheme host = url.hostname port = url.port self.test(host, int(port)) def test(self, address, port): import socket s = socket.socket() s.settimeout(1) s.connect((address, port)) @trap_timeout def getWorkspacesNames(self): return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs()) def workspaceExists(self, name): return name in self.getWorkspacesNames() @trap_timeout def addWorkspace(self, aWorkspace): self.__serv.create_db(aWorkspace.lower()) return self.__getDb(aWorkspace) @trap_timeout def addDocument(self, aWorkspaceName, documentId, aDocument): self.incrementSeqNumber(aWorkspaceName) self.__getDb(aWorkspaceName)[documentId] = aDocument @trap_timeout def saveDocument(self, aWorkspaceName, aDocument): self.incrementSeqNumber(aWorkspaceName) model.api.log("Saving document in remote workspace %s" % aWorkspaceName) self.__getDb(aWorkspaceName).save_doc(aDocument, use_uuids=True, force_update=True) @trap_timeout def __getDb(self, aWorkspaceName): aWorkspaceName = aWorkspaceName.lower() model.api.log("Getting workspace [%s]" % aWorkspaceName) workspacedb = self.__dbs.get(aWorkspaceName, self.__serv.get_db(aWorkspaceName)) if not self.__dbs.has_key(aWorkspaceName): model.api.log("Asking couchdb for workspace [%s]" % aWorkspaceName) self.__dbs[aWorkspaceName] = workspacedb self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq'] return workspacedb @trap_timeout def getDocument(self, aWorkspaceName, documentId): model.api.log("Getting document for workspace [%s]" % aWorkspaceName) return self.__getDb(aWorkspaceName).get(documentId) @trap_timeout def checkDocument(self, aWorkspaceName, documentName): return self.__getDb(aWorkspaceName).doc_exist(documentName) @trap_timeout def replicate(self, workspace, *targets_dbs, **kwargs): model.api.log("Targets to replicate %s" % str(targets_dbs)) for target_db in targets_dbs: src_db_path = "/".join([self.__uri, workspace]) dst_db_path = "/".join([target_db, workspace]) try: model.api.devlog( "workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs)) self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs) except ResourceNotFound as e: raise e except Exception as e: model.api.devlog(e) raise def __peerReplication(self, workspace, src, dst, **kwargs): mutual = kwargs.get("mutual", True) continuous = kwargs.get("continuous", True) ct = kwargs.get("create_target", True) self.__serv.replicate(workspace, dst, mutual=mutual, continuous=continuous, create_target=ct) if mutual: self.__serv.replicate(dst, src, continuous=continuous, **kwargs) def getLastChangeSeq(self, workspaceName): self.mutex.acquire() seq = self.__seq_nums[workspaceName] self.mutex.release() return seq def setLastChangeSeq(self, workspaceName, seq_num): self.mutex.acquire() self.__seq_nums[workspaceName] = seq_num self.mutex.release() @trap_timeout def waitForDBChange(self, db_name, since=0, timeout=15000): """ Be warned this will return after the database has a change, if there was one before call it will return immediatly with the changes done""" changes = [] last_seq = max(self.getLastChangeSeq(db_name), since) db = self.__getDb(db_name) with ChangesStream(db, feed="longpoll", since=last_seq, timeout=timeout) as stream: for change in stream: if change['seq'] > self.getLastChangeSeq(db_name): changes.append(change) last_seq = reduce(lambda x, y: max(y['seq'], x), changes, self.getLastChangeSeq(db_name)) self.setLastChangeSeq(db_name, last_seq) return changes @trap_timeout def delete_all_dbs(self): for db in self.__serv.all_dbs(): self.__serv.delete_db(db) @trap_timeout def existWorkspace(self, name): return name in self.__serv.all_dbs() @trap_timeout def workspaceDocumentsIterator(self, workspaceName): return filter(lambda x: not x["id"].startswith("_"), self.__getDb(workspaceName).documents(include_docs=True)) @trap_timeout def removeWorkspace(self, workspace_name): return self.__serv.delete_db(workspace_name) @trap_timeout def remove(self, workspace, host_id): self.incrementSeqNumber(workspace) self.__dbs[workspace].delete_doc(host_id) @trap_timeout def compactDatabase(self, aWorkspaceName): self.__getDb(aWorkspaceName).compact() def pushReports(self): vmanager = ViewsManager() reports = os.path.join(os.getcwd(), "views", "reports") workspace = self.__serv.get_or_create_db("reports") vmanager.addView(reports, workspace) return self.__uri + "/reports/_design/reports/index.html" def addViews(self, workspaceName): vmanager = ViewsManager() workspace = self.__getDb(workspaceName) for v in vmanager.getAvailableViews(): vmanager.addView(v, workspace) def getViews(self, workspaceName): vmanager = ViewsManager() workspace = self.__getDb(workspaceName) return vmanager.getViews(workspace) def syncWorkspaceViews(self, workspaceName): vmanager = ViewsManager() workspace = self.__getDb(workspaceName) installed_views = vmanager.getViews(workspace) for v in vmanager.getAvailableViews(): if v not in installed_views: vmanager.addView(v, workspace) def incrementSeqNumber(self, workspaceName): self.mutex.acquire() self.__seq_nums[workspaceName] += 1 self.mutex.release()
class CouchDbManager(AbstractPersistenceManager): """ This is a couchdb manager for the workspace, it will load from the couchdb databases """ def __init__(self, uri): super(CouchDbManager, self).__init__() getLogger(self).debug( "Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self.__serv = NoConectionServer() self._available = False try: if uri is not None: self.testCouchUrl(uri) url = urlparse(uri) getLogger(self).debug( "Setting user,pass %s %s" % (url.username, url.password)) self.__serv = Server(uri=uri) self.__serv.resource_class.credentials = (url.username, url.password) self._available = True self.pushReports() self._loadDbs() except: getLogger(self).warn("No route to couchdb server on: %s" % uri) getLogger(self).debug(traceback.format_exc()) #@trap_timeout def _create(self, name): db = self.__serv.create_db(name.lower()) return CouchDbConnector(db) #@trap_timeout def _delete(self, name): self.__serv.delete_db(name) #@trap_timeout def _loadDbs(self): conditions = lambda x: not x.startswith("_") and x != 'reports' for dbname in filter(conditions, self.__serv.all_dbs()): if dbname not in self.dbs.keys(): getLogger(self).debug( "Asking for dbname[%s], registering for lazy initialization" % dbname) self.dbs[dbname] = lambda x: self._loadDb(x) def _loadDb(self, dbname): db = self.__serv.get_db(dbname) seq = db.info()['update_seq'] self.dbs[dbname] = CouchDbConnector(db, seq_num=seq) return self.dbs[dbname] #@trap_timeout def pushReports(self): vmanager = ViewsManager() reports = os.path.join(os.getcwd(), "views", "reports") workspace = self.__serv.get_or_create_db("reports") vmanager.addView(reports, workspace) return self.__uri + "/reports/_design/reports/index.html" def lostConnectionResolv(self): self._lostConnection = True self.__dbs.clear() self.__serv = NoConectionServer() def reconnect(self): ret_val = False ur = self.__uri if CouchDbManager.testCouch(ur): self.__serv = Server(uri = ur) self.__dbs.clear() self._lostConnection = False ret_val = True return ret_val @staticmethod def testCouch(uri): if uri is not None: host, port = None, None try: import socket url = urlparse(uri) proto = url.scheme host = url.hostname port = url.port port = port if port else socket.getservbyname(proto) s = socket.socket() s.settimeout(1) s.connect((host, int(port))) except: return False #getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port)) return True def testCouchUrl(self, uri): if uri is not None: url = urlparse(uri) proto = url.scheme host = url.hostname port = url.port self.test(host, int(port)) def test(self, address, port): import socket s = socket.socket() s.settimeout(1) s.connect((address, port)) #@trap_timeout def replicate(self, workspace, *targets_dbs, **kwargs): getLogger(self).debug("Targets to replicate %s" % str(targets_dbs)) for target_db in targets_dbs: src_db_path = "/".join([self.__uri, workspace]) dst_db_path = "/".join([target_db, workspace]) try: getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs)) self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs) except ResourceNotFound as e: raise e except Exception as e: getLogger(self).error(e) raise def __peerReplication(self, workspace, src, dst, **kwargs): mutual = kwargs.get("mutual", True) continuous = kwargs.get("continuous", True) ct = kwargs.get("create_target", True) self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct) if mutual: self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class CouchdbManager(PersistenceManager): """ This is a couchdb manager for the workspace, it will load from the couchdb databases""" def __init__(self, uri): self._last_seq_ack = 0 getLogger(self).debug("Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self.__dbs = {} self.__seq_nums = {} self.__serv = NoConectionServer() self.mutex = threading.Lock() self._available = False #setting the doc types to load from couch def get_types(subclasses): if len(subclasses): head = subclasses[0] tail = [] if len(subclasses[1:]): tail = subclasses[1:] return get_types(head.__subclasses__()) + [head.class_signature] + get_types(tail) return [] self._model_object_types = get_types([ModelObject]) try: if uri is not None: self.testCouchUrl(uri) url = urlparse(uri) getLogger(self).debug("Setting user,pass %s %s" % (url.username, url.password)) self.__serv = Server(uri=uri) #print dir(self.__serv) self.__serv.resource_class.credentials = (url.username, url.password) self._available = True except: getLogger(self).warn("No route to couchdb server on: %s" % uri) getLogger(self).debug(traceback.format_exc()) def isAvailable(self): return self._available def lostConnectionResolv(self): self._lostConnection = True self.__dbs.clear() self.__serv = NoConectionServer() def reconnect(self): ret_val = False ur = self.__uri if CouchdbManager.testCouch(ur): self.__serv = Server(uri = ur) self.__dbs.clear() self._lostConnection = False ret_val = True return ret_val @staticmethod def testCouch(uri): if uri is not None: host, port = None, None try: import socket url = urlparse(uri) proto = url.scheme host = url.hostname port = url.port port = port if port else socket.getservbyname(proto) s = socket.socket() s.settimeout(1) s.connect((host, int(port))) except: return False getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port)) return True def testCouchUrl(self, uri): if uri is not None: url = urlparse(uri) proto = url.scheme host = url.hostname port = url.port self.test(host, int(port)) def test(self, address, port): import socket s = socket.socket() s.settimeout(1) s.connect((address, port)) @trap_timeout def getWorkspacesNames(self): return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs()) def workspaceExists(self, name): return name in self.getWorkspacesNames() @trap_timeout def addWorkspace(self, aWorkspace): self.__serv.create_db(aWorkspace.lower()) return self._getDb(aWorkspace) @trap_timeout def addDocument(self, aWorkspaceName, documentId, aDocument): self._getDb(aWorkspaceName) self.incrementSeqNumber(aWorkspaceName) self._getDb(aWorkspaceName)[documentId] = aDocument @trap_timeout def saveDocument(self, aWorkspaceName, aDocument): self.incrementSeqNumber(aWorkspaceName) getLogger(self).debug("Saving document in remote workspace %s" % aWorkspaceName) return self._getDb(aWorkspaceName).save_doc(aDocument, use_uuids = True, force_update = True) def _getDb(self, aWorkspaceName): if not self.__dbs.has_key(aWorkspaceName): self.__getDb(aWorkspaceName) return self.__dbs.get(aWorkspaceName, None) @trap_timeout def __getDb(self, aWorkspaceName): aWorkspaceName = aWorkspaceName.lower() getLogger(self).debug("Getting workspace [%s]" % aWorkspaceName) workspacedb = self.__dbs.get(aWorkspaceName, self.__serv.get_db(aWorkspaceName)) if not self.__dbs.has_key(aWorkspaceName): getLogger(self).debug("Asking couchdb for workspace [%s]" % aWorkspaceName) self.__dbs[aWorkspaceName] = workspacedb self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq'] return workspacedb @trap_timeout def getDocument(self, aWorkspaceName, documentId): getLogger(self).debug("Getting document for workspace [%s]" % aWorkspaceName) return self._getDb(aWorkspaceName).get(documentId) @trap_timeout def getDeletedDocument(self, aWorkspaceName, documentId, documentRev): return self._getDb(aWorkspaceName).get(documentId, rev=documentRev) @trap_timeout def checkDocument(self, aWorkspaceName, documentName): return self._getDb(aWorkspaceName).doc_exist(documentName) @trap_timeout def replicate(self, workspace, *targets_dbs, **kwargs): getLogger(self).debug("Targets to replicate %s" % str(targets_dbs)) for target_db in targets_dbs: src_db_path = "/".join([self.__uri, workspace]) dst_db_path = "/".join([target_db, workspace]) try: getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs)) self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs) except ResourceNotFound as e: raise e except Exception as e: getLogger(self).error(e) raise def __peerReplication(self, workspace, src, dst, **kwargs): mutual = kwargs.get("mutual", True) continuous = kwargs.get("continuous", True) ct = kwargs.get("create_target", True) self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct) if mutual: self.__serv.replicate(dst, src, continuous = continuous, **kwargs) def getLastChangeSeq(self, workspaceName): self.mutex.acquire() seq = self.__seq_nums[workspaceName] self.mutex.release() return seq def setLastChangeSeq(self, workspaceName, seq_num): self.mutex.acquire() self.__seq_nums[workspaceName] = seq_num self.mutex.release() @trap_timeout def waitForDBChange(self, db_name, since = 0, timeout = 15000): """ Be warned this will return after the database has a change, if there was one before call it will return immediatly with the changes done""" changes = [] last_seq = max(self.getLastChangeSeq(db_name), since) db = self._getDb(db_name) with ChangesStream(db, feed="longpoll", since=last_seq, timeout=timeout) as stream: for change in stream: if change['seq'] > self.getLastChangeSeq(db_name): self.setLastChangeSeq(db_name, change['seq']) if not change['id'].startswith('_design'): #fake doc type for deleted objects doc = {'type': 'unknown', '_deleted': 'False', '_rev':[0]} if not change.get('deleted'): doc = self.getDocument(db_name, change['id']) changes.append(change_factory.create(doc)) if len(changes): getLogger(self).debug("Changes from another instance") return changes @trap_timeout def delete_all_dbs(self): for db in self.__serv.all_dbs(): self.__serv.delete_db(db) @trap_timeout def existWorkspace(self, name): return name in self.__serv.all_dbs() @trap_timeout def workspaceDocumentsIterator(self, workspaceName): return filter(self.filterConditions, self._getDb(workspaceName).documents(include_docs=True)) def filterConditions(self, doc): ret = True ret = ret and not doc["id"].startswith("_") ret = ret and doc['doc']["type"] in self._model_object_types return ret @trap_timeout def removeWorkspace(self, workspace_name): return self.__serv.delete_db(workspace_name) @trap_timeout def remove(self, workspace, host_id): self.incrementSeqNumber(workspace) self.__dbs[workspace].delete_doc(host_id) @trap_timeout def compactDatabase(self, aWorkspaceName): self._getDb(aWorkspaceName).compact() def pushReports(self): vmanager = ViewsManager() reports = os.path.join(os.getcwd(), "views", "reports") workspace = self.__serv.get_or_create_db("reports") vmanager.addView(reports, workspace) return self.__uri + "/reports/_design/reports/index.html" def addViews(self, workspaceName): vmanager = ViewsManager() workspace = self._getDb(workspaceName) for v in vmanager.getAvailableViews(): vmanager.addView(v, workspace) def getViews(self, workspaceName): vmanager = ViewsManager() workspace = self._getDb(workspaceName) return vmanager.getViews(workspace) def syncWorkspaceViews(self, workspaceName): vmanager = ViewsManager() workspace = self._getDb(workspaceName) installed_views = vmanager.getViews(workspace) for v in vmanager.getAvailableViews(): if v not in installed_views: vmanager.addView(v, workspace) def incrementSeqNumber(self, workspaceName): self.mutex.acquire() if not self.__seq_nums.has_key(workspaceName): self.__seq_nums[workspaceName] = 0 self.__seq_nums[workspaceName] += 1 self.mutex.release()
config = ConfigParser.ConfigParser() config.read('config.ini') #Config ConsumerKey = config.get('Twitter', 'ConsumerKey', 0) ConsumerSecret = config.get('Twitter', 'ConsumerSecret', 0) AccessToken = config.get('Twitter', 'AccessToken', 0) AccessSecret = config.get('Twitter', 'AccessSecret', 0) TweetDbName = config.get('Twitter', 'DbName', 0) ReweetDbName = config.get('Twitter', 'ReweetDbName', 0) Filter = config.get('Twitter', 'Filter', 0) UserIds = config.get('Twitter', 'UserIds', 0) Languages = config.get('Twitter', 'Languages', 0) Tweetdb = CouchDbServer.create_db(TweetDbName) ReTweetdb = CouchDbServer.create_db(ReweetDbName) #Print Red def print_red(text): print ("\033[91m {}\033[00m" .format(text)) #Print Green def print_green(text): print ("\033[92m {}\033[00m" .format(text)) #PRint Yellow def print_yellow(text): print ("\033[93m {}\033[00m" .format(text)) class listener(StreamListener): def on_data(self, data): #Load Json
class DbConnectorCouchTestSuite(unittest.TestCase): def setUp(self): self.couch_srv = Server(uri=CONF.getCouchURI()) self.db_name = new_random_workspace_name() self.db = self.couch_srv.create_db(self.db_name) def tearDown(self): self.couch_srv.delete_db(self.db_name) time.sleep(3) def test_save_Document(self): couchConnector = CouchDbConnector(self.db) doc = {"_id": "123", "data": "some data"} couchConnector.saveDocument(doc) doc_from_db = self.db.get("123") self.assertNotEquals(doc_from_db, None, "Document should be retrieved") self.assertEquals(doc_from_db.get("data"), "some data", "Data retrieved should be the same as data saved") def test_get_Document(self): couchConnector = CouchDbConnector(self.db) doc = {"_id": "123", "data": "some data"} couchConnector.saveDocument(doc) doc_retrieved = couchConnector.getDocument("123") self.assertNotEquals(doc_retrieved, None, "Document should be retrieved") self.assertEquals(doc_retrieved.get("data"), "some data", "Data retrieved should be the same as data saved") def test_remove_Document(self): couchConnector = CouchDbConnector(self.db) doc = {"_id": "123", "data": "some data"} couchConnector.saveDocument(doc) couchConnector.remove("123") try: doc_from_db = self.db.get("123") except ResourceNotFound: doc_from_db = None self.assertEquals(doc_from_db, None, "Document should be None") def test_get_by_parent_and_type(self): couchConnector = CouchDbConnector(self.db) doc = {"_id": "123", "type": "father", "parent": None} couchConnector.saveDocument(doc) doc = {"_id": "456", "type": "child", "parent": "123"} couchConnector.saveDocument(doc) doc = {"_id": "789", "type": "child", "parent": "123"} couchConnector.saveDocument(doc) ids = couchConnector.getDocsByFilter(parentId="123", type="child") self.assertEquals(len(ids), 2, "There should be two 'childs' with parent '123'") self.assertIn("456", ids, "Child '456' should be in the list of childs") self.assertIn("789", ids, "Child '789' should be in the list of childs") ids = couchConnector.getDocsByFilter(parentId="123", type="son") self.assertEquals(len(ids), 0, "There shouldn't be any 'son' with parent '123'") ids = couchConnector.getDocsByFilter(parentId="456", type="child") self.assertEquals(len(ids), 0, "There shouldn't be any 'child' with parent '456'")
class CouchDbManager(AbstractPersistenceManager): """ This is a couchdb manager for the workspace, it will load from the couchdb databases """ def __init__(self, uri, couch_exception_callback): super(CouchDbManager, self).__init__() getLogger(self).debug( "Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self._available = False self.couch_exception_callback = couch_exception_callback test_couch_thread = threading.Thread(target=self.continuosly_check_connection) test_couch_thread.daemon = True test_couch_thread.start() try: if uri is not None: self.testCouchUrl(uri) url = urlparse(uri) getLogger(self).debug( "Setting user,pass %s %s" % (url.username, url.password)) self.__serv = Server(uri=uri) self.__serv.resource_class.credentials = (url.username, url.password) self._available = True self.pushReports() self._loadDbs() except: getLogger(self).warn("No route to couchdb server on: %s" % uri) getLogger(self).debug(traceback.format_exc()) def continuosly_check_connection(self): """Intended to use on a separate thread. Call module-level function testCouch every second to see if response to the server_uri of the DB is still 200. Call the exception_callback if we can't access the server three times in a row. """ tolerance = 0 server_uri = self.__uri while True: time.sleep(1) test_was_successful = test_couch(server_uri) if test_was_successful: tolerance = 0 else: tolerance += 1 if tolerance == 3: self.couch_exception_callback() return False # kill the thread if something went wrong def _create(self, name): db = self.__serv.create_db(name.lower()) return CouchDbConnector(db) def _delete(self, name): self.__serv.delete_db(name) def _loadDbs(self): def conditions(database): begins_with_underscore = database.startswith("_") is_blacklisted = database in CONST_BLACKDBS return not begins_with_underscore and not is_blacklisted try: for dbname in filter(conditions, self.__serv.all_dbs()): if dbname not in self.dbs.keys(): getLogger(self).debug( "Asking for dbname[%s], registering for lazy initialization" % dbname) self.dbs[dbname] = lambda x: self._loadDb(x) except restkit.errors.RequestError as req_error: getLogger(self).error("Couldn't load databases. " "The connection to the CouchDB was probably lost. ") def _loadDb(self, dbname): db = self.__serv.get_db(dbname) seq = db.info()['update_seq'] self.dbs[dbname] = CouchDbConnector(db, seq_num=seq) return self.dbs[dbname] def refreshDbs(self): """Refresh databases using inherited method. On exception, asume no databases are available. """ try: return AbstractPersistenceManager.refreshDbs() except: return [] def pushReports(self): vmanager = ViewsManager() reports = os.path.join(os.getcwd(), "views", "reports") try: workspace = self.__serv.get_or_create_db("reports") vmanager.addView(reports, workspace) except: getLogger(self).warn( "Reports database couldn't be uploaded. You need to be an admin to do it") return self.__uri + "/reports/_design/reports/index.html" @staticmethod def testCouch(uri): """Redirect to the module-level function of the name, which serves the same purpose and is used by other classes too.""" return test_couch(uri) def testCouchUrl(self, uri): if uri is not None: url = urlparse(uri) host = url.hostname port = url.port self.test(host, int(port)) def test(self, address, port): import socket s = socket.socket() s.settimeout(1) s.connect((address, port)) def replicate(self, workspace, *targets_dbs, **kwargs): getLogger(self).debug("Targets to replicate %s" % str(targets_dbs)) for target_db in targets_dbs: src_db_path = "/".join([self.__uri, workspace]) dst_db_path = "/".join([target_db, workspace]) try: getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs)) self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs) except ResourceNotFound as e: raise e except Exception as e: getLogger(self).error(e) raise def __peerReplication(self, workspace, src, dst, **kwargs): mutual = kwargs.get("mutual", True) continuous = kwargs.get("continuous", True) ct = kwargs.get("create_target", True) self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct) if mutual: self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class CouchdbManager(PersistenceManager): """ This is a couchdb manager for the workspace, it will load from the couchdb databases""" def __init__(self, uri): self._last_seq_ack = 0 model.api.log("Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self.__dbs = {} self.__seq_nums = {} self.__serv = NoConectionServer() self.mutex = threading.Lock() self._available = False try: self.testCouchUrl(uri) self.__serv = Server(uri = uri) self._available = True except: model.api.log("No route to couchdb server on: %s" % uri) def isAvailable(self): return self._available def lostConnectionResolv(self): self._lostConnection = True self.__dbs.clear() self.__serv = NoConectionServer() def reconnect(self): ret_val = False ur = self.__uri if CouchdbManager.testCouch(ur): self.__serv = Server(uri = ur) self.__dbs.clear() self._lostConnection = False ret_val = True return ret_val @staticmethod def testCouch(uri): host, port = None, None try: import socket proto, netloc, _, _, _ = urlsplit(uri) host, port = splitport(netloc) port = port if port else socket.getservbyname(proto) s = socket.socket() s.settimeout(1) s.connect((host, int(port))) except: return False model.api.log("Connecting Couch to: %s:%s" % (host, port)) return True def testCouchUrl(self, uri): _, netloc, _, _, _ = urlsplit(uri) host, port = splitport(netloc) self.test(host, int(port)) def test(self, address, port): import socket s = socket.socket() s.settimeout(1) s.connect((address, port)) @trap_timeout def getWorkspacesNames(self): return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs()) def workspaceExists(self, name): return name in self.getWorkspacesNames() @trap_timeout def addWorkspace(self, aWorkspace): self.__serv.create_db(aWorkspace.lower()) return self.__getDb(aWorkspace) @trap_timeout def addDocument(self, aWorkspaceName, documentId, aDocument): self.incrementSeqNumber(aWorkspaceName) self.__getDb(aWorkspaceName)[documentId] = aDocument @trap_timeout def saveDocument(self, aWorkspaceName, aDocument): self.incrementSeqNumber(aWorkspaceName) model.api.log("Saving document in remote workspace %s" % aWorkspaceName) self.__getDb(aWorkspaceName).save_doc(aDocument, use_uuids = True, force_update = True) @trap_timeout def __getDb(self, aWorkspaceName): aWorkspaceName = aWorkspaceName.lower() model.api.log("Getting workspace [%s]" % aWorkspaceName) workspacedb = self.__dbs.get(aWorkspaceName, self.__serv.get_db(aWorkspaceName)) if not self.__dbs.has_key(aWorkspaceName): model.api.log("Asking couchdb for workspace [%s]" % aWorkspaceName) self.__dbs[aWorkspaceName] = workspacedb self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq'] return workspacedb @trap_timeout def getDocument(self, aWorkspaceName, documentId): model.api.log("Getting document for workspace [%s]" % aWorkspaceName) return self.__getDb(aWorkspaceName).get(documentId) @trap_timeout def checkDocument(self, aWorkspaceName, documentName): return self.__getDb(aWorkspaceName).doc_exist(documentName) @trap_timeout def replicate(self, workspace, *targets_dbs, **kwargs): model.api.log("Targets to replicate %s" % str(targets_dbs)) for target_db in targets_dbs: src_db_path = "/".join([self.__uri, workspace]) dst_db_path = "/".join([target_db, workspace]) try: model.api.devlog("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs)) self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs) except ResourceNotFound as e: raise e except Exception as e: model.api.devlog(e) raise def __peerReplication(self, workspace, src, dst, **kwargs): mutual = kwargs.get("mutual", True) continuous = kwargs.get("continuous", True) ct = kwargs.get("create_target", True) self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct) if mutual: self.__serv.replicate(dst, src, continuous = continuous, **kwargs) def getLastChangeSeq(self, workspaceName): self.mutex.acquire() seq = self.__seq_nums[workspaceName] self.mutex.release() return seq def setLastChangeSeq(self, workspaceName, seq_num): self.mutex.acquire() self.__seq_nums[workspaceName] = seq_num self.mutex.release() @trap_timeout def waitForDBChange(self, db_name, since = 0, timeout = 15000): """ Be warned this will return after the database has a change, if there was one before call it will return immediatly with the changes done""" changes = [] last_seq = max(self.getLastChangeSeq(db_name), since) db = self.__getDb(db_name) with ChangesStream(db, feed="longpoll", since = last_seq, timeout = timeout) as stream: for change in stream: if change['seq'] > self.getLastChangeSeq(db_name): changes.append(change) last_seq = reduce(lambda x,y: max(y['seq'], x) , changes, self.getLastChangeSeq(db_name)) self.setLastChangeSeq(db_name, last_seq) return changes @trap_timeout def delete_all_dbs(self): for db in self.__serv.all_dbs(): self.__serv.delete_db(db) @trap_timeout def existWorkspace(self, name): return name in self.__serv.all_dbs() @trap_timeout def workspaceDocumentsIterator(self, workspaceName): return filter(lambda x: not x["id"].startswith("_"), self.__getDb(workspaceName).documents(include_docs=True)) @trap_timeout def removeWorkspace(self, workspace_name): return self.__serv.delete_db(workspace_name) @trap_timeout def remove(self, workspace, host_id): self.incrementSeqNumber(workspace) self.__dbs[workspace].delete_doc(host_id) @trap_timeout def compactDatabase(self, aWorkspaceName): self.__getDb(aWorkspaceName).compact() def pushReports(self): vmanager = ViewsManager() reports = os.path.join(os.getcwd(), "views", "reports") workspace = self.__serv.get_or_create_db("reports") vmanager.addView(reports, workspace) return self.__uri + "/reports/_design/reports/index.html" def addViews(self, workspaceName): vmanager = ViewsManager() workspace = self.__getDb(workspaceName) for v in vmanager.getAvailableViews(): vmanager.addView(v, workspace) def getViews(self, workspaceName): vmanager = ViewsManager() workspace = self.__getDb(workspaceName) return vmanager.getViews(workspace) def syncWorkspaceViews(self, workspaceName): vmanager = ViewsManager() workspace = self.__getDb(workspaceName) installed_views = vmanager.getViews(workspace) for v in vmanager.getAvailableViews(): if v not in installed_views: vmanager.addView(v, workspace) def incrementSeqNumber(self, workspaceName): self.mutex.acquire() self.__seq_nums[workspaceName] += 1 self.mutex.release()
class CouchjockTestCase(unittest2.TestCase): server_url = 'http://localhost:5984/' db_name = 'couchjock__test' schema = couchjock def setUp(self): self.server = Server(uri=self.server_url) self.db = self.server.create_db(self.db_name) def tearDown(self): self.server.delete_db(self.db_name) def test_save(self): class Foo(self.schema.Document): _db = self.db pass foo = Foo() foo.save() foo_id = foo._id self.assertIsNotNone(foo_id) foo2 = Foo.get(foo_id) self.assertEqual(foo2._id, foo_id) def test_simple_schema(self): class Foo(self.schema.Document): _db = self.db string = self.schema.StringProperty() boolean = self.schema.BooleanProperty(default=True) foo1 = Foo() foo1.save() foo1_id = foo1._id foo1_rev = foo1._rev self.assertIsNotNone(foo1_id) self.assertIsNotNone(foo1_rev) foo1 = Foo.get(foo1_id) self.assertEqual(foo1.to_json(), { 'doc_type': 'Foo', '_id': foo1_id, '_rev': foo1_rev, 'string': None, 'boolean': True, }) foo1._doc.update({'boolean': False}) self.assertEqual(foo1.boolean, False) def _individual_save(self, docs): for doc in docs: doc.save() def _bulk_save(self, docs): self.db.bulk_save(docs) def _test_simple_view(self, save_fn): class Foo(self.schema.Document): _db = self.db string = self.schema.StringProperty() foo1 = Foo(string='fun one') foo2 = Foo(string='poop') save_fn([foo1, foo2]) self.assertEqual( map(lambda x: x.to_json(), Foo.view('_all_docs', include_docs=True).all()), map(lambda x: x.to_json(), sorted([foo1, foo2], key=attrgetter('_id'))) ) def test_simple_view(self): self._test_simple_view(self._individual_save) def test_bulk_save(self): self._test_simple_view(self._bulk_save)
class ClientDatabaseTestCase(unittest.TestCase): def setUp(self): self.couchdb = CouchdbResource() self.Server = Server() def tearDown(self): try: del self.Server["couchdbkit_test"] except: pass def testCreateDatabase(self): db = self.Server.create_db("couchdbkit_test") self.assertIsInstance(db, Database) info = db.info() self.assertEqual(info["db_name"], "couchdbkit_test") del self.Server["couchdbkit_test"] def testDbFromUri(self): db = self.Server.create_db("couchdbkit_test") db1 = Database("http://127.0.0.1:5984/couchdbkit_test") self.assertIs(hasattr(db1, "dbname"), True) self.assertEqual(db1.dbname, "couchdbkit_test") info = db1.info() self.assertEqual(info["db_name"], "couchdbkit_test") def testCreateEmptyDoc(self): db = self.Server.create_db("couchdbkit_test") doc = {} db.save_doc(doc) del self.Server["couchdbkit_test"] self.assertIn("_id", doc) def testCreateDoc(self): db = self.Server.create_db("couchdbkit_test") # create doc without id doc = {"string": "test", "number": 4} db.save_doc(doc) self.assertTrue(db.doc_exist(doc["_id"])) # create doc with id doc1 = {"_id": "test", "string": "test", "number": 4} db.save_doc(doc1) self.assertTrue(db.doc_exist("test")) doc2 = {"string": "test", "number": 4} db["test2"] = doc2 self.assertTrue(db.doc_exist("test2")) del self.Server["couchdbkit_test"] db = self.Server.create_db("couchdbkit/test") doc1 = {"_id": "test", "string": "test", "number": 4} db.save_doc(doc1) self.assertTrue(db.doc_exist("test")) del self.Server["couchdbkit/test"] def testUpdateDoc(self): db = self.Server.create_db("couchdbkit_test") doc = {"string": "test", "number": 4} db.save_doc(doc) doc.update({"number": 6}) db.save_doc(doc) doc = db.get(doc["_id"]) self.assertEqual(doc["number"], 6) del self.Server["couchdbkit_test"] def testDocWithSlashes(self): db = self.Server.create_db("couchdbkit_test") doc = {"_id": "a/b"} db.save_doc(doc) self.assertIn("a/b", db) doc = {"_id": "http://a"} db.save_doc(doc) self.assertIn("http://a", db) doc = db.get("http://a") self.assertIsNotNone(doc) def not_found(): doc = db.get("http:%2F%2Fa") self.assertRaises(ResourceNotFound, not_found) self.assertEqual("http://a", doc.get("_id")) doc.get("_id") doc = {"_id": "http://b"} db.save_doc(doc) self.assertIn("http://b", db) doc = {"_id": "_design/a"} db.save_doc(doc) self.assertIn("_design/a", db) del self.Server["couchdbkit_test"] def testGetRev(self): db = self.Server.create_db("couchdbkit_test") doc = {} db.save_doc(doc) rev = db.get_rev(doc["_id"]) self.assertEqual(doc["_rev"], rev) def testForceUpdate(self): db = self.Server.create_db("couchdbkit_test") doc = {} db.save_doc(doc) doc1 = doc.copy() db.save_doc(doc) self.assertRaises(ResourceConflict, db.save_doc, doc1) try: db.save_doc(doc1, force_update=True) except ResourceConflict: is_conflict = True else: is_conflict = False self.assertIs(is_conflict, False) def testMultipleDocWithSlashes(self): db = self.Server.create_db("couchdbkit_test") doc = {"_id": "a/b"} doc1 = {"_id": "http://a"} doc3 = {"_id": "_design/a"} db.bulk_save([doc, doc1, doc3]) self.assertIn("a/b", db) self.assertIn("http://a", db) self.assertIn("_design/a", db) def not_found(): doc = db.get("http:%2F%2Fa") self.assertRaises(ResourceNotFound, not_found) def testFlush(self): db = self.Server.create_db("couchdbkit_test") doc1 = {"_id": "test", "string": "test", "number": 4} db.save_doc(doc1) doc2 = {"string": "test", "number": 4} db["test2"] = doc2 self.assertTrue(db.doc_exist("test")) self.assertTrue(db.doc_exist("test2")) design_doc = { "_id": "_design/test", "language": "javascript", "views": { "all": { "map": """function(doc) { if (doc.docType == "test") { emit(doc._id, doc); }}""" } }, } db.save_doc(design_doc) self.assertEqual(len(db), 3) db.flush() self.assertEqual(len(db), 1) self.assertFalse(db.doc_exist("test")) self.assertFalse(db.doc_exist("test2")) self.assertTrue(db.doc_exist("_design/test")) ddoc = db.get("_design/test") self.assertIn("all", ddoc["views"]) del self.Server["couchdbkit_test"] def testDbLen(self): db = self.Server.create_db("couchdbkit_test") doc1 = {"string": "test", "number": 4} db.save_doc(doc1) doc2 = {"string": "test2", "number": 4} db.save_doc(doc2) self.assertEqual(len(db), 2) del self.Server["couchdbkit_test"] def testDeleteDoc(self): db = self.Server.create_db("couchdbkit_test") doc = {"string": "test", "number": 4} db.save_doc(doc) docid = doc["_id"] db.delete_doc(docid) self.assertIs(db.doc_exist(docid), False) doc = {"string": "test", "number": 4} db.save_doc(doc) docid = doc["_id"] db.delete_doc(doc) self.assertIs(db.doc_exist(docid), False) del self.Server["couchdbkit_test"] def testStatus404(self): db = self.Server.create_db("couchdbkit_test") def no_doc(): doc = db.get("t") self.assertRaises(ResourceNotFound, no_doc) del self.Server["couchdbkit_test"] def testInlineAttachments(self): db = self.Server.create_db("couchdbkit_test") attachment = "<html><head><title>test attachment</title></head><body><p>Some words</p></body></html>" doc = { "_id": "docwithattachment", "f": "value", "_attachments": {"test.html": {"type": "text/html", "data": attachment}}, } db.save_doc(doc) fetch_attachment = db.fetch_attachment(doc, "test.html") self.assertEqual(attachment, fetch_attachment) doc1 = db.get("docwithattachment") self.assertIn("_attachments", doc1) self.assertIn("test.html", doc1["_attachments"]) self.assertIn("stub", doc1["_attachments"]["test.html"]) self.assertIs(doc1["_attachments"]["test.html"]["stub"], True) self.assertEqual(len(attachment), doc1["_attachments"]["test.html"]["length"]) del self.Server["couchdbkit_test"] def testMultipleInlineAttachments(self): db = self.Server.create_db("couchdbkit_test") attachment = "<html><head><title>test attachment</title></head><body><p>Some words</p></body></html>" attachment2 = "<html><head><title>test attachment</title></head><body><p>More words</p></body></html>" doc = { "_id": "docwithattachment", "f": "value", "_attachments": { "test.html": {"type": "text/html", "data": attachment}, "test2.html": {"type": "text/html", "data": attachment2}, }, } db.save_doc(doc) fetch_attachment = db.fetch_attachment(doc, "test.html") self.assertEqual(attachment, fetch_attachment) fetch_attachment = db.fetch_attachment(doc, "test2.html") self.assertEqual(attachment2, fetch_attachment) doc1 = db.get("docwithattachment") self.assertIn("test.html", doc1["_attachments"]) self.assertIn("test2.html", doc1["_attachments"]) self.assertEqual(len(attachment), doc1["_attachments"]["test.html"]["length"]) self.assertEqual(len(attachment2), doc1["_attachments"]["test2.html"]["length"]) del self.Server["couchdbkit_test"] def testInlineAttachmentWithStub(self): db = self.Server.create_db("couchdbkit_test") attachment = "<html><head><title>test attachment</title></head><body><p>Some words</p></body></html>" attachment2 = "<html><head><title>test attachment</title></head><body><p>More words</p></body></html>" doc = { "_id": "docwithattachment", "f": "value", "_attachments": {"test.html": {"type": "text/html", "data": attachment}}, } db.save_doc(doc) doc1 = db.get("docwithattachment") doc1["_attachments"].update({"test2.html": {"type": "text/html", "data": attachment2}}) db.save_doc(doc1) fetch_attachment = db.fetch_attachment(doc1, "test2.html") self.assertEqual(attachment2, fetch_attachment) doc2 = db.get("docwithattachment") self.assertIn("test.html", doc2["_attachments"]) self.assertIn("test2.html", doc2["_attachments"]) self.assertEqual(len(attachment), doc2["_attachments"]["test.html"]["length"]) self.assertEqual(len(attachment2), doc2["_attachments"]["test2.html"]["length"]) del self.Server["couchdbkit_test"] def testAttachments(self): db = self.Server.create_db("couchdbkit_test") doc = {"string": "test", "number": 4} db.save_doc(doc) text_attachment = u"un texte attaché" old_rev = doc["_rev"] db.put_attachment(doc, text_attachment, "test", "text/plain") self.assertNotEqual(old_rev, doc["_rev"]) fetch_attachment = db.fetch_attachment(doc, "test") self.assertEqual(text_attachment, fetch_attachment) del self.Server["couchdbkit_test"] def testFetchAttachmentStream(self): db = self.Server.create_db("couchdbkit_test") doc = {"string": "test", "number": 4} db.save_doc(doc) text_attachment = u"a text attachment" db.put_attachment(doc, text_attachment, "test", "text/plain") stream = db.fetch_attachment(doc, "test", stream=True) fetch_attachment = stream.read() self.assertEqual(text_attachment, fetch_attachment) del self.Server["couchdbkit_test"] def testEmptyAttachment(self): db = self.Server.create_db("couchdbkit_test") doc = {} db.save_doc(doc) db.put_attachment(doc, "", name="test") doc1 = db.get(doc["_id"]) attachment = doc1["_attachments"]["test"] self.assertEqual(0, attachment["length"]) del self.Server["couchdbkit_test"] def testDeleteAttachment(self): db = self.Server.create_db("couchdbkit_test") doc = {"string": "test", "number": 4} db.save_doc(doc) text_attachment = "un texte attaché" old_rev = doc["_rev"] db.put_attachment(doc, text_attachment, "test", "text/plain") db.delete_attachment(doc, "test") self.assertRaises(ResourceNotFound, db.fetch_attachment, doc, "test") del self.Server["couchdbkit_test"] def testAttachmentsWithSlashes(self): db = self.Server.create_db("couchdbkit_test") doc = {"_id": "test/slashes", "string": "test", "number": 4} db.save_doc(doc) text_attachment = u"un texte attaché" old_rev = doc["_rev"] db.put_attachment(doc, text_attachment, "test", "text/plain") self.assertNotEqual(old_rev, doc["_rev"]) fetch_attachment = db.fetch_attachment(doc, "test") self.assertEqual(text_attachment, fetch_attachment) db.put_attachment(doc, text_attachment, "test/test.txt", "text/plain") self.assertNotEqual(old_rev, doc["_rev"]) fetch_attachment = db.fetch_attachment(doc, "test/test.txt") self.assertEqual(text_attachment, fetch_attachment) del self.Server["couchdbkit_test"] def testAttachmentUnicode8URI(self): db = self.Server.create_db("couchdbkit_test") doc = {"_id": u"éàù/slashes", "string": "test", "number": 4} db.save_doc(doc) text_attachment = u"un texte attaché" old_rev = doc["_rev"] db.put_attachment(doc, text_attachment, "test", "text/plain") self.assertNotEqual(old_rev, doc["_rev"]) fetch_attachment = db.fetch_attachment(doc, "test") self.assertEqual(text_attachment, fetch_attachment) del self.Server["couchdbkit_test"] def testSaveMultipleDocs(self): db = self.Server.create_db("couchdbkit_test") docs = [ {"string": "test", "number": 4}, {"string": "test", "number": 5}, {"string": "test", "number": 4}, {"string": "test", "number": 6}, ] db.bulk_save(docs) self.assertEqual(len(db), 4) self.assertIn("_id", docs[0]) self.assertIn("_rev", docs[0]) doc = db.get(docs[2]["_id"]) self.assertEqual(doc["number"], 4) docs[3]["number"] = 6 old_rev = docs[3]["_rev"] db.bulk_save(docs) self.assertNotEqual(docs[3]["_rev"], old_rev) doc = db.get(docs[3]["_id"]) self.assertEqual(doc["number"], 6) docs = [ {"_id": "test", "string": "test", "number": 4}, {"string": "test", "number": 5}, {"_id": "test2", "string": "test", "number": 42}, {"string": "test", "number": 6}, ] db.bulk_save(docs) doc = db.get("test2") self.assertEqual(doc["number"], 42) del self.Server["couchdbkit_test"] def testDeleteMultipleDocs(self): db = self.Server.create_db("couchdbkit_test") docs = [ {"string": "test", "number": 4}, {"string": "test", "number": 5}, {"string": "test", "number": 4}, {"string": "test", "number": 6}, ] db.bulk_save(docs) self.assertEqual(len(db), 4) db.bulk_delete(docs) self.assertEqual(len(db), 0) self.assertEqual(db.info()["doc_del_count"], 4) del self.Server["couchdbkit_test"] def testMultipleDocCOnflict(self): db = self.Server.create_db("couchdbkit_test") docs = [ {"string": "test", "number": 4}, {"string": "test", "number": 5}, {"string": "test", "number": 4}, {"string": "test", "number": 6}, ] db.bulk_save(docs) self.assertEqual(len(db), 4) docs1 = [ docs[0], docs[1], {"_id": docs[2]["_id"], "string": "test", "number": 4}, {"_id": docs[3]["_id"], "string": "test", "number": 6}, ] self.assertRaises(BulkSaveError, db.bulk_save, docs1) docs2 = [ docs1[0], docs1[1], {"_id": docs[2]["_id"], "string": "test", "number": 4}, {"_id": docs[3]["_id"], "string": "test", "number": 6}, ] doc23 = docs2[3].copy() all_errors = [] try: db.bulk_save(docs2) except BulkSaveError, e: all_errors = e.errors self.assertEqual(len(all_errors), 2) self.assertEqual(all_errors[0]["error"], "conflict") self.assertEqual(doc23, docs2[3]) docs3 = [ docs2[0], docs2[1], {"_id": docs[2]["_id"], "string": "test", "number": 4}, {"_id": docs[3]["_id"], "string": "test", "number": 6}, ] doc33 = docs3[3].copy() all_errors2 = [] try: db.bulk_save(docs3, all_or_nothing=True) except BulkSaveError, e: all_errors2 = e.errors
class ClientViewTestCase(unittest.TestCase): def setUp(self): self.couchdb = CouchdbResource() self.Server = Server() def tearDown(self): try: del self.Server["couchdbkit_test"] except: pass try: self.Server.delete_db("couchdbkit_test2") except: pass def testView(self): db = self.Server.create_db("couchdbkit_test") # save 2 docs doc1 = {"_id": "test", "string": "test", "number": 4, "docType": "test"} db.save_doc(doc1) doc2 = {"_id": "test2", "string": "test", "number": 2, "docType": "test"} db.save_doc(doc2) design_doc = { "_id": "_design/test", "language": "javascript", "views": { "all": { "map": """function(doc) { if (doc.docType == "test") { emit(doc._id, doc); }}""" } }, } db.save_doc(design_doc) doc3 = db.get("_design/test") self.assertIsNotNone(doc3) results = db.view("test/all") self.assertEqual(len(results), 2) del self.Server["couchdbkit_test"] def testAllDocs(self): db = self.Server.create_db("couchdbkit_test") # save 2 docs doc1 = {"_id": "test", "string": "test", "number": 4, "docType": "test"} db.save_doc(doc1) doc2 = {"_id": "test2", "string": "test", "number": 2, "docType": "test"} db.save_doc(doc2) self.assertEqual(db.view("_all_docs").count(), 2) self.assertEqual(db.view("_all_docs").all(), db.all_docs().all()) del self.Server["couchdbkit_test"] def testCount(self): db = self.Server.create_db("couchdbkit_test") # save 2 docs doc1 = {"_id": "test", "string": "test", "number": 4, "docType": "test"} db.save_doc(doc1) doc2 = {"_id": "test2", "string": "test", "number": 2, "docType": "test"} db.save_doc(doc2) design_doc = { "_id": "_design/test", "language": "javascript", "views": {"all": {"map": """function(doc) { if (doc.docType == "test") { emit(doc._id, doc); }}"""}}, } db.save_doc(design_doc) count = db.view("/test/all").count() self.assertEqual(count, 2) del self.Server["couchdbkit_test"] def testTemporaryView(self): db = self.Server.create_db("couchdbkit_test") # save 2 docs doc1 = {"_id": "test", "string": "test", "number": 4, "docType": "test"} db.save_doc(doc1) doc2 = {"_id": "test2", "string": "test", "number": 2, "docType": "test"} db.save_doc(doc2) design_doc = { "map": """function(doc) { if (doc.docType == "test") { emit(doc._id, doc); }}""" } results = db.temp_view(design_doc) self.assertEqual(len(results), 2) del self.Server["couchdbkit_test"] def testView2(self): db = self.Server.create_db("couchdbkit_test") # save 2 docs doc1 = {"_id": "test1", "string": "test", "number": 4, "docType": "test"} db.save_doc(doc1) doc2 = {"_id": "test2", "string": "test", "number": 2, "docType": "test"} db.save_doc(doc2) doc3 = {"_id": "test3", "string": "test", "number": 2, "docType": "test2"} db.save_doc(doc3) design_doc = { "_id": "_design/test", "language": "javascript", "views": { "with_test": { "map": """function(doc) { if (doc.docType == "test") { emit(doc._id, doc); }}""" }, "with_test2": { "map": """function(doc) { if (doc.docType == "test2") { emit(doc._id, doc); }}""" }, }, } db.save_doc(design_doc) # yo view is callable ! results = db.view("test/with_test") self.assertEqual(len(results), 2) results = db.view("test/with_test2") self.assertEqual(len(results), 1) del self.Server["couchdbkit_test"] def testViewWithParams(self): db = self.Server.create_db("couchdbkit_test") # save 2 docs doc1 = {"_id": "test1", "string": "test", "number": 4, "docType": "test", "date": "20081107"} db.save_doc(doc1) doc2 = {"_id": "test2", "string": "test", "number": 2, "docType": "test", "date": "20081107"} db.save_doc(doc2) doc3 = {"_id": "test3", "string": "machin", "number": 2, "docType": "test", "date": "20081007"} db.save_doc(doc3) doc4 = {"_id": "test4", "string": "test2", "number": 2, "docType": "test", "date": "20081108"} db.save_doc(doc4) doc5 = {"_id": "test5", "string": "test2", "number": 2, "docType": "test", "date": "20081109"} db.save_doc(doc5) doc6 = {"_id": "test6", "string": "test2", "number": 2, "docType": "test", "date": "20081109"} db.save_doc(doc6) design_doc = { "_id": "_design/test", "language": "javascript", "views": { "test1": { "map": """function(doc) { if (doc.docType == "test") { emit(doc.string, doc); }}""" }, "test2": { "map": """function(doc) { if (doc.docType == "test") { emit(doc.date, doc); }}""" }, "test3": { "map": """function(doc) { if (doc.docType == "test") { emit(doc.string, doc); }}""" }, }, } db.save_doc(design_doc) results = db.view("test/test1") self.assertEqual(len(results), 6) results = db.view("test/test3", key="test") self.assertEqual(len(results), 2) results = db.view("test/test3", key="test2") self.assertEqual(len(results), 3) results = db.view("test/test2", startkey="200811") self.assertEqual(len(results), 5) results = db.view("test/test2", startkey="20081107", endkey="20081108") self.assertEqual(len(results), 3) results = db.view("test/test1", keys=["test", "machin"]) self.assertEqual(len(results), 3) del self.Server["couchdbkit_test"] def testMultiWrap(self): """ Tests wrapping of view results to multiple classes using the client """ class A(Document): pass class B(Document): pass design_doc = { "_id": "_design/test", "language": "javascript", "views": {"all": {"map": """function(doc) { emit(doc._id, doc); }"""}}, } a = A() a._id = "1" b = B() b._id = "2" db = self.Server.create_db("couchdbkit_test") A._db = db B._db = db a.save() b.save() db.save_doc(design_doc) # provide classes as a list results = list(db.view("test/all", schema=[A, B])) self.assertEqual(results[0].__class__, A) self.assertEqual(results[1].__class__, B) # provide classes as a dict results = list(db.view("test/all", schema={"A": A, "B": B})) self.assertEqual(results[0].__class__, A) self.assertEqual(results[1].__class__, B) self.Server.delete_db("couchdbkit_test")
class ClientServerTestCase(unittest.TestCase): def setUp(self): self.couchdb = CouchdbResource() self.Server = Server() def tearDown(self): try: del self.Server["couchdbkit_test"] del self.Server["couchdbkit/test"] except: pass def testGetInfo(self): info = self.Server.info() self.assertIn("version", info) def testCreateDb(self): res = self.Server.create_db("couchdbkit_test") self.assertIsInstance(res, Database) all_dbs = self.Server.all_dbs() self.assertIn("couchdbkit_test", all_dbs) del self.Server["couchdbkit_test"] res = self.Server.create_db("couchdbkit/test") self.assertIn("couchdbkit/test", self.Server.all_dbs()) del self.Server["couchdbkit/test"] def testGetOrCreateDb(self): # create the database gocdb = self.Server.get_or_create_db("get_or_create_db") self.assertEqual(gocdb.dbname, "get_or_create_db") self.assertIn("get_or_create_db", self.Server) self.Server.delete_db("get_or_create_db") # get the database (already created) self.assertNotIn("get_or_create_db", self.Server) db = self.Server.create_db("get_or_create_db") self.assertIn("get_or_create_db", self.Server) gocdb = self.Server.get_or_create_db("get_or_create_db") self.assertEqual(db.dbname, gocdb.dbname) self.Server.delete_db("get_or_create_db") def testCreateInvalidDbName(self): def create_invalid(): res = self.Server.create_db("123ab") self.assertRaises(ValueError, create_invalid) def testServerLen(self): res = self.Server.create_db("couchdbkit_test") self.assertGreaterEqual(len(self.Server), 1) self.assertTrue(self.Server) del self.Server["couchdbkit_test"] def testServerContain(self): res = self.Server.create_db("couchdbkit_test") self.assertIn("couchdbkit_test", self.Server) del self.Server["couchdbkit_test"] def testGetUUIDS(self): uuid = self.Server.next_uuid() self.assertIsInstance(uuid, basestring) self.assertEqual(len(self.Server._uuids), 999) uuid2 = self.Server.next_uuid() self.assertNotEqual(uuid, uuid2) self.assertEqual(len(self.Server._uuids), 998)
class CouchDbManager(AbstractPersistenceManager): """ This is a couchdb manager for the workspace, it will load from the couchdb databases """ def __init__(self, uri): super(CouchDbManager, self).__init__() getLogger(self).debug( "Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self.__serv = NoConectionServer() self._available = False try: if uri is not None: self.testCouchUrl(uri) url = urlparse(uri) getLogger(self).debug( "Setting user,pass %s %s" % (url.username, url.password)) self.__serv = Server(uri=uri) self.__serv.resource_class.credentials = (url.username, url.password) self._available = True self.pushReports() self._loadDbs() except: getLogger(self).warn("No route to couchdb server on: %s" % uri) getLogger(self).debug(traceback.format_exc()) #@trap_timeout def _create(self, name): db = self.__serv.create_db(name.lower()) return CouchDbConnector(db) #@trap_timeout def _delete(self, name): self.__serv.delete_db(name) #@trap_timeout def _loadDbs(self): conditions = lambda x: not x.startswith("_") and x != 'reports' for dbname in filter(conditions, self.__serv.all_dbs()): if dbname not in self.dbs.keys(): getLogger(self).debug( "Asking for dbname[%s], registering for lazy initialization" % dbname) self.dbs[dbname] = lambda x: self._loadDb(x) def _loadDb(self, dbname): db = self.__serv.get_db(dbname) seq = db.info()['update_seq'] self.dbs[dbname] = CouchDbConnector(db, seq_num=seq) return self.dbs[dbname] #@trap_timeout def pushReports(self): vmanager = ViewsManager() reports = os.path.join(os.getcwd(), "views", "reports") workspace = self.__serv.get_or_create_db("reports") vmanager.addView(reports, workspace) return self.__uri + "/reports/_design/reports/index.html" def lostConnectionResolv(self): self._lostConnection = True self.__dbs.clear() self.__serv = NoConectionServer() def reconnect(self): ret_val = False ur = self.__uri if CouchDbManager.testCouch(ur): self.__serv = Server(uri = ur) self.__dbs.clear() self._lostConnection = False ret_val = True return ret_val @staticmethod def testCouch(uri): if uri is not None: host, port = None, None try: import socket url = urlparse(uri) proto = url.scheme host = url.hostname port = url.port port = port if port else socket.getservbyname(proto) s = socket.socket() s.settimeout(1) s.connect((host, int(port))) except: return False #getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port)) return True def testCouchUrl(self, uri): if uri is not None: url = urlparse(uri) proto = url.scheme host = url.hostname port = url.port self.test(host, int(port)) def test(self, address, port): import socket s = socket.socket() s.settimeout(1) s.connect((address, port)) #@trap_timeout def replicate(self, workspace, *targets_dbs, **kwargs): getLogger(self).debug("Targets to replicate %s" % str(targets_dbs)) for target_db in targets_dbs: src_db_path = "/".join([self.__uri, workspace]) dst_db_path = "/".join([target_db, workspace]) try: getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs)) self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs) except ResourceNotFound as e: raise e except Exception as e: getLogger(self).error(e) raise def __peerReplication(self, workspace, src, dst, **kwargs): mutual = kwargs.get("mutual", True) continuous = kwargs.get("continuous", True) ct = kwargs.get("create_target", True) self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct) if mutual: self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class CouchDbManager(AbstractPersistenceManager): """ This is a couchdb manager for the workspace, it will load from the couchdb databases """ def __init__(self, uri): super(CouchDbManager, self).__init__() getLogger(self).debug("Initializing CouchDBManager for url [%s]" % uri) self._lostConnection = False self.__uri = uri self.__serv = NoConectionServer() self._available = False try: if uri is not None: self.testCouchUrl(uri) url = urlparse(uri) getLogger(self).debug("Setting user,pass %s %s" % (url.username, url.password)) self.__serv = Server(uri=uri) self.__serv.resource_class.credentials = (url.username, url.password) self._available = True self.pushReports() self._loadDbs() except: getLogger(self).warn("No route to couchdb server on: %s" % uri) getLogger(self).debug(traceback.format_exc()) #@trap_timeout def _create(self, name): db = self.__serv.create_db(name.lower()) return CouchDbConnector(db) #@trap_timeout def _delete(self, name): self.__serv.delete_db(name) #@trap_timeout def _loadDbs(self): conditions = lambda x: not x.startswith("_" ) and x not in CONST_BLACKDBS try: for dbname in filter(conditions, self.__serv.all_dbs()): if dbname not in self.dbs.keys(): getLogger(self).debug( "Asking for dbname[%s], registering for lazy initialization" % dbname) self.dbs[dbname] = lambda x: self._loadDb(x) except restkit.errors.RequestError as req_error: getLogger(self).error( "Couldn't load databases. " "The connection to the CouchDB was probably lost. ") def _loadDb(self, dbname): db = self.__serv.get_db(dbname) seq = db.info()['update_seq'] self.dbs[dbname] = CouchDbConnector(db, seq_num=seq) return self.dbs[dbname] def refreshDbs(self): """Refresh databases using inherited method. On exception, asume no databases are available. """ try: return AbstractPersistenceManager.refreshDbs() except: return [] #@trap_timeout def pushReports(self): vmanager = ViewsManager() reports = os.path.join(os.getcwd(), "views", "reports") try: workspace = self.__serv.get_or_create_db("reports") vmanager.addView(reports, workspace) except: getLogger(self).warn( "Reports database couldn't be uploaded. You need to be an admin to do it" ) return self.__uri + "/reports/_design/reports/index.html" def lostConnectionResolv(self): self._lostConnection = True self.__dbs.clear() self.__serv = NoConectionServer() def reconnect(self): ret_val = False ur = self.__uri if CouchDbManager.testCouch(ur): self.__serv = Server(uri=ur) self.__dbs.clear() self._lostConnection = False ret_val = True return ret_val @staticmethod def testCouch(uri): """Redirect to the module-level function of the name, which serves the same purpose and is used by other classes too.""" return test_couch(uri) def testCouchUrl(self, uri): if uri is not None: url = urlparse(uri) proto = url.scheme host = url.hostname port = url.port self.test(host, int(port)) def test(self, address, port): import socket s = socket.socket() s.settimeout(1) s.connect((address, port)) #@trap_timeout def replicate(self, workspace, *targets_dbs, **kwargs): getLogger(self).debug("Targets to replicate %s" % str(targets_dbs)) for target_db in targets_dbs: src_db_path = "/".join([self.__uri, workspace]) dst_db_path = "/".join([target_db, workspace]) try: getLogger(self).info( "workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs)) self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs) except ResourceNotFound as e: raise e except Exception as e: getLogger(self).error(e) raise def __peerReplication(self, workspace, src, dst, **kwargs): mutual = kwargs.get("mutual", True) continuous = kwargs.get("continuous", True) ct = kwargs.get("create_target", True) self.__serv.replicate(workspace, dst, mutual=mutual, continuous=continuous, create_target=ct) if mutual: self.__serv.replicate(dst, src, continuous=continuous, **kwargs)