class CouchDBServer(object):
def __init__(self):
self.__get_server_uri()
self.__authenticate()
self.__connect()
def __get_server_uri(self):
couchdb_port = config.couchdb.port if config.couchdb.protocol == 'http' else config.couchdb.ssl_port
self.__couchdb_uri = "%s://%s:%s" % (config.couchdb.protocol, config.couchdb.host, couchdb_port)
def __authenticate(self):
user, passwd = config.couchdb.user, config.couchdb.password
if all((user, passwd)):
auth = restkit.BasicAuth(user, passwd)
self.__auth_resource = CouchdbResource(filters=[auth])
else:
self.__auth_resource = None
def __connect(self):
self.__server = Server(uri=self.__couchdb_uri, resource_instance=self.__auth_resource)
def list_workspaces(self):
return filter(is_usable_workspace, self.__server.all_dbs())
def get_workspace_handler(self, ws_name):
return self.__server.get_db(ws_name)
def get_or_create_db(self, ws_name):
return self.__server.get_or_create_db(ws_name)
def test_create_and_get_db(self):
couch_manager = CouchDbManager(uri=CONF.getCouchURI())
couch_manager.createDb(self.dbname)
self.assertNotEquals(
couch_manager.getDb(self.dbname),
None,
"Db %s shouldn't be None" % self.dbname)
server = Server(uri=CONF.getCouchURI())
self.assertIn(
self.dbname,
server.all_dbs(),
"Db %s should be in the db list" % self.dbname)
def test_delete_db(self):
couch_manager = CouchDbManager(uri=CONF.getCouchURI())
couch_manager.createDb(self.dbname)
self.assertNotEquals(couch_manager.getDb(self.dbname), None,
"Db %s shouldn't be None" % self.dbname)
couch_manager.deleteDb(self.dbname)
self.assertEquals(couch_manager.getDb(self.dbname), None,
"Db %s should be None" % self.dbname)
server = Server(uri=CONF.getCouchURI())
self.assertNotIn(self.dbname, server.all_dbs(),
"Db %s shouldn't be in the db list" % self.dbname)
def index(request):
server = Server(settings.COUCHDB_SERVER)
databases = [server.get_or_create_db(db).info() for db in server.all_dbs()]
create_database_form = CreateDatabaseForm(request.POST or None)
if create_database_form.is_valid():
database_name = create_database_form.cleaned_data["name"]
return HttpResponseRedirect(reverse("cushion_database",
args=(database_name,)))
return render_to_response("cushion/index.html",
{"title": "CouchDB",
"server": server,
"databases": databases,
"form": create_database_form},
context_instance=RequestContext(request))
class CouchDBServer(object):
def __init__(self):
self.__get_server_uri()
self.__authenticate()
self.__connect()
def __get_server_uri(self):
couchdb_port = config.couchdb.port if config.couchdb.protocol == 'http' else config.couchdb.ssl_port
self.__couchdb_uri = "%s://%s:%s" % (config.couchdb.protocol,
config.couchdb.host, couchdb_port)
def __authenticate(self):
user, passwd = config.couchdb.user, config.couchdb.password
if all((user, passwd)):
auth = restkit.BasicAuth(user, passwd)
self.__auth_resource = CouchdbResource(filters=[auth])
else:
self.__auth_resource = None
def __connect(self):
self.__server = Server(uri=self.__couchdb_uri,
resource_instance=self.__auth_resource)
def list_workspaces(self):
return filter(is_usable_workspace, self.__server.all_dbs())
def get_workspace_handler(self, ws_name):
return self.__server.get_db(ws_name)
def get_or_create_db(self, ws_name):
return self.__server.get_or_create_db(ws_name)
def create_db(self, ws_name):
return self.__server.create_db(ws_name)
def delete_db(self, ws_name):
return self.__server.delete_db(ws_name)
if not server_urls:
logging.error("Please supply one or more URL's for CouchDB instances to carry out maintenance for.")
sys.exit(1)
for server_url in server_urls:
try:
server = Server(server_url)
except:
logging.error("Could not connect to CouchDB at %s", server_url)
continue
logging.info("Connected to CouchDB server at %s", server_url)
logging.info("Getting list of databases...")
try:
db_names = server.all_dbs()
except:
logging.error("Could not get list of databases for CouchDB at %s", server_url)
continue
logging.info("Databases: %s", ", ".join(db_names))
for db_name in db_names:
if db_name.startswith('_'):
logging.info("Skipping database starting with '_': %s", db_name)
continue
logging.info("Database: %s", db_name)
db = server.get_or_create_db(db_name)
logging.info("Starting compaction for database %s...", db_name)
try:
class CouchdbManager(PersistenceManager):
""" This is a couchdb manager for the workspace, it will load from the
couchdb databases"""
def __init__(self, uri):
self._last_seq_ack = 0
model.api.log("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
try:
self.testCouchUrl(uri)
url = urlparse(uri)
print("Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
#print dir(self.__serv)
self.__serv.resource_class.credentials = (url.username,
url.password)
self._available = True
except:
model.api.log("No route to couchdb server on: %s" % uri)
print(traceback.format_exc())
def isAvailable(self):
return self._available
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchdbManager.testCouch(ur):
self.__serv = Server(uri=ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
model.api.log("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
@trap_timeout
def getWorkspacesNames(self):
return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs())
def workspaceExists(self, name):
return name in self.getWorkspacesNames()
@trap_timeout
def addWorkspace(self, aWorkspace):
self.__serv.create_db(aWorkspace.lower())
return self.__getDb(aWorkspace)
@trap_timeout
def addDocument(self, aWorkspaceName, documentId, aDocument):
self.incrementSeqNumber(aWorkspaceName)
self.__getDb(aWorkspaceName)[documentId] = aDocument
@trap_timeout
def saveDocument(self, aWorkspaceName, aDocument):
self.incrementSeqNumber(aWorkspaceName)
model.api.log("Saving document in remote workspace %s" %
aWorkspaceName)
self.__getDb(aWorkspaceName).save_doc(aDocument,
use_uuids=True,
force_update=True)
@trap_timeout
def __getDb(self, aWorkspaceName):
aWorkspaceName = aWorkspaceName.lower()
model.api.log("Getting workspace [%s]" % aWorkspaceName)
workspacedb = self.__dbs.get(aWorkspaceName,
self.__serv.get_db(aWorkspaceName))
if not self.__dbs.has_key(aWorkspaceName):
model.api.log("Asking couchdb for workspace [%s]" % aWorkspaceName)
self.__dbs[aWorkspaceName] = workspacedb
self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq']
return workspacedb
@trap_timeout
def getDocument(self, aWorkspaceName, documentId):
model.api.log("Getting document for workspace [%s]" % aWorkspaceName)
return self.__getDb(aWorkspaceName).get(documentId)
@trap_timeout
def checkDocument(self, aWorkspaceName, documentName):
return self.__getDb(aWorkspaceName).doc_exist(documentName)
@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
model.api.log("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
model.api.devlog(
"workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s"
% (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path,
**kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
model.api.devlog(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace,
dst,
mutual=mutual,
continuous=continuous,
create_target=ct)
if mutual:
self.__serv.replicate(dst, src, continuous=continuous, **kwargs)
def getLastChangeSeq(self, workspaceName):
self.mutex.acquire()
seq = self.__seq_nums[workspaceName]
self.mutex.release()
return seq
def setLastChangeSeq(self, workspaceName, seq_num):
self.mutex.acquire()
self.__seq_nums[workspaceName] = seq_num
self.mutex.release()
@trap_timeout
def waitForDBChange(self, db_name, since=0, timeout=15000):
""" Be warned this will return after the database has a change, if
there was one before call it will return immediatly with the changes
done"""
changes = []
last_seq = max(self.getLastChangeSeq(db_name), since)
db = self.__getDb(db_name)
with ChangesStream(db,
feed="longpoll",
since=last_seq,
timeout=timeout) as stream:
for change in stream:
if change['seq'] > self.getLastChangeSeq(db_name):
changes.append(change)
last_seq = reduce(lambda x, y: max(y['seq'], x), changes,
self.getLastChangeSeq(db_name))
self.setLastChangeSeq(db_name, last_seq)
return changes
@trap_timeout
def delete_all_dbs(self):
for db in self.__serv.all_dbs():
self.__serv.delete_db(db)
@trap_timeout
def existWorkspace(self, name):
return name in self.__serv.all_dbs()
@trap_timeout
def workspaceDocumentsIterator(self, workspaceName):
return filter(lambda x: not x["id"].startswith("_"),
self.__getDb(workspaceName).documents(include_docs=True))
@trap_timeout
def removeWorkspace(self, workspace_name):
return self.__serv.delete_db(workspace_name)
@trap_timeout
def remove(self, workspace, host_id):
self.incrementSeqNumber(workspace)
self.__dbs[workspace].delete_doc(host_id)
@trap_timeout
def compactDatabase(self, aWorkspaceName):
self.__getDb(aWorkspaceName).compact()
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def addViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
for v in vmanager.getAvailableViews():
vmanager.addView(v, workspace)
def getViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
return vmanager.getViews(workspace)
def syncWorkspaceViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
installed_views = vmanager.getViews(workspace)
for v in vmanager.getAvailableViews():
if v not in installed_views:
vmanager.addView(v, workspace)
def incrementSeqNumber(self, workspaceName):
self.mutex.acquire()
self.__seq_nums[workspaceName] += 1
self.mutex.release()
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri):
super(CouchDbManager, self).__init__()
getLogger(self).debug(
"Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__serv = NoConectionServer()
self._available = False
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug(
"Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
#@trap_timeout
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
#@trap_timeout
def _delete(self, name):
self.__serv.delete_db(name)
#@trap_timeout
def _loadDbs(self):
conditions = lambda x: not x.startswith("_") and x != 'reports'
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization" % dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
#@trap_timeout
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchDbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
if uri is not None:
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
#getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
#@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class CouchdbManager(PersistenceManager):
""" This is a couchdb manager for the workspace, it will load from the
couchdb databases"""
def __init__(self, uri):
self._last_seq_ack = 0
getLogger(self).debug("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
#setting the doc types to load from couch
def get_types(subclasses):
if len(subclasses):
head = subclasses[0]
tail = []
if len(subclasses[1:]):
tail = subclasses[1:]
return get_types(head.__subclasses__()) + [head.class_signature] + get_types(tail)
return []
self._model_object_types = get_types([ModelObject])
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug("Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
#print dir(self.__serv)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
def isAvailable(self):
return self._available
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchdbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
if uri is not None:
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
@trap_timeout
def getWorkspacesNames(self):
return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs())
def workspaceExists(self, name):
return name in self.getWorkspacesNames()
@trap_timeout
def addWorkspace(self, aWorkspace):
self.__serv.create_db(aWorkspace.lower())
return self._getDb(aWorkspace)
@trap_timeout
def addDocument(self, aWorkspaceName, documentId, aDocument):
self._getDb(aWorkspaceName)
self.incrementSeqNumber(aWorkspaceName)
self._getDb(aWorkspaceName)[documentId] = aDocument
@trap_timeout
def saveDocument(self, aWorkspaceName, aDocument):
self.incrementSeqNumber(aWorkspaceName)
getLogger(self).debug("Saving document in remote workspace %s" % aWorkspaceName)
return self._getDb(aWorkspaceName).save_doc(aDocument, use_uuids = True, force_update = True)
def _getDb(self, aWorkspaceName):
if not self.__dbs.has_key(aWorkspaceName):
self.__getDb(aWorkspaceName)
return self.__dbs.get(aWorkspaceName, None)
@trap_timeout
def __getDb(self, aWorkspaceName):
aWorkspaceName = aWorkspaceName.lower()
getLogger(self).debug("Getting workspace [%s]" % aWorkspaceName)
workspacedb = self.__dbs.get(aWorkspaceName, self.__serv.get_db(aWorkspaceName))
if not self.__dbs.has_key(aWorkspaceName):
getLogger(self).debug("Asking couchdb for workspace [%s]" % aWorkspaceName)
self.__dbs[aWorkspaceName] = workspacedb
self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq']
return workspacedb
@trap_timeout
def getDocument(self, aWorkspaceName, documentId):
getLogger(self).debug("Getting document for workspace [%s]" % aWorkspaceName)
return self._getDb(aWorkspaceName).get(documentId)
@trap_timeout
def getDeletedDocument(self, aWorkspaceName, documentId, documentRev):
return self._getDb(aWorkspaceName).get(documentId, rev=documentRev)
@trap_timeout
def checkDocument(self, aWorkspaceName, documentName):
return self._getDb(aWorkspaceName).doc_exist(documentName)
@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
def getLastChangeSeq(self, workspaceName):
self.mutex.acquire()
seq = self.__seq_nums[workspaceName]
self.mutex.release()
return seq
def setLastChangeSeq(self, workspaceName, seq_num):
self.mutex.acquire()
self.__seq_nums[workspaceName] = seq_num
self.mutex.release()
@trap_timeout
def waitForDBChange(self, db_name, since = 0, timeout = 15000):
""" Be warned this will return after the database has a change, if
there was one before call it will return immediatly with the changes
done"""
changes = []
last_seq = max(self.getLastChangeSeq(db_name), since)
db = self._getDb(db_name)
with ChangesStream(db, feed="longpoll", since=last_seq, timeout=timeout) as stream:
for change in stream:
if change['seq'] > self.getLastChangeSeq(db_name):
self.setLastChangeSeq(db_name, change['seq'])
if not change['id'].startswith('_design'):
#fake doc type for deleted objects
doc = {'type': 'unknown', '_deleted': 'False', '_rev':[0]}
if not change.get('deleted'):
doc = self.getDocument(db_name, change['id'])
changes.append(change_factory.create(doc))
if len(changes):
getLogger(self).debug("Changes from another instance")
return changes
@trap_timeout
def delete_all_dbs(self):
for db in self.__serv.all_dbs():
self.__serv.delete_db(db)
@trap_timeout
def existWorkspace(self, name):
return name in self.__serv.all_dbs()
@trap_timeout
def workspaceDocumentsIterator(self, workspaceName):
return filter(self.filterConditions, self._getDb(workspaceName).documents(include_docs=True))
def filterConditions(self, doc):
ret = True
ret = ret and not doc["id"].startswith("_")
ret = ret and doc['doc']["type"] in self._model_object_types
return ret
@trap_timeout
def removeWorkspace(self, workspace_name):
return self.__serv.delete_db(workspace_name)
@trap_timeout
def remove(self, workspace, host_id):
self.incrementSeqNumber(workspace)
self.__dbs[workspace].delete_doc(host_id)
@trap_timeout
def compactDatabase(self, aWorkspaceName):
self._getDb(aWorkspaceName).compact()
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def addViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self._getDb(workspaceName)
for v in vmanager.getAvailableViews():
vmanager.addView(v, workspace)
def getViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self._getDb(workspaceName)
return vmanager.getViews(workspace)
def syncWorkspaceViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self._getDb(workspaceName)
installed_views = vmanager.getViews(workspace)
for v in vmanager.getAvailableViews():
if v not in installed_views:
vmanager.addView(v, workspace)
def incrementSeqNumber(self, workspaceName):
self.mutex.acquire()
if not self.__seq_nums.has_key(workspaceName):
self.__seq_nums[workspaceName] = 0
self.__seq_nums[workspaceName] += 1
self.mutex.release()
def tearDown(self):
server = Server(uri=CONF.getCouchURI())
if self.dbname in server.all_dbs():
server.delete_db(self.dbname)
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri, couch_exception_callback):
super(CouchDbManager, self).__init__()
getLogger(self).debug(
"Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self._available = False
self.couch_exception_callback = couch_exception_callback
test_couch_thread = threading.Thread(target=self.continuosly_check_connection)
test_couch_thread.daemon = True
test_couch_thread.start()
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug(
"Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
def continuosly_check_connection(self):
"""Intended to use on a separate thread. Call module-level
function testCouch every second to see if response to the server_uri
of the DB is still 200. Call the exception_callback if we can't access
the server three times in a row.
"""
tolerance = 0
server_uri = self.__uri
while True:
time.sleep(1)
test_was_successful = test_couch(server_uri)
if test_was_successful:
tolerance = 0
else:
tolerance += 1
if tolerance == 3:
self.couch_exception_callback()
return False # kill the thread if something went wrong
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
def _delete(self, name):
self.__serv.delete_db(name)
def _loadDbs(self):
def conditions(database):
begins_with_underscore = database.startswith("_")
is_blacklisted = database in CONST_BLACKDBS
return not begins_with_underscore and not is_blacklisted
try:
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization" % dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
except restkit.errors.RequestError as req_error:
getLogger(self).error("Couldn't load databases. "
"The connection to the CouchDB was probably lost. ")
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
def refreshDbs(self):
"""Refresh databases using inherited method. On exception, asume
no databases are available.
"""
try:
return AbstractPersistenceManager.refreshDbs()
except:
return []
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
try:
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
except:
getLogger(self).warn(
"Reports database couldn't be uploaded. You need to be an admin to do it")
return self.__uri + "/reports/_design/reports/index.html"
@staticmethod
def testCouch(uri):
"""Redirect to the module-level function of the name, which
serves the same purpose and is used by other classes too."""
return test_couch(uri)
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class ClientServerTestCase(unittest.TestCase):
def setUp(self):
self.couchdb = CouchdbResource()
self.Server = Server()
def tearDown(self):
try:
del self.Server["couchdbkit_test"]
del self.Server["couchdbkit/test"]
except:
pass
def testGetInfo(self):
info = self.Server.info()
self.assertIn("version", info)
def testCreateDb(self):
res = self.Server.create_db("couchdbkit_test")
self.assertIsInstance(res, Database)
all_dbs = self.Server.all_dbs()
self.assertIn("couchdbkit_test", all_dbs)
del self.Server["couchdbkit_test"]
res = self.Server.create_db("couchdbkit/test")
self.assertIn("couchdbkit/test", self.Server.all_dbs())
del self.Server["couchdbkit/test"]
def testGetOrCreateDb(self):
# create the database
gocdb = self.Server.get_or_create_db("get_or_create_db")
self.assertEqual(gocdb.dbname, "get_or_create_db")
self.assertIn("get_or_create_db", self.Server)
self.Server.delete_db("get_or_create_db")
# get the database (already created)
self.assertNotIn("get_or_create_db", self.Server)
db = self.Server.create_db("get_or_create_db")
self.assertIn("get_or_create_db", self.Server)
gocdb = self.Server.get_or_create_db("get_or_create_db")
self.assertEqual(db.dbname, gocdb.dbname)
self.Server.delete_db("get_or_create_db")
def testCreateInvalidDbName(self):
def create_invalid():
res = self.Server.create_db("123ab")
self.assertRaises(ValueError, create_invalid)
def testServerLen(self):
res = self.Server.create_db("couchdbkit_test")
self.assertGreaterEqual(len(self.Server), 1)
self.assertTrue(self.Server)
del self.Server["couchdbkit_test"]
def testServerContain(self):
res = self.Server.create_db("couchdbkit_test")
self.assertIn("couchdbkit_test", self.Server)
del self.Server["couchdbkit_test"]
def testGetUUIDS(self):
uuid = self.Server.next_uuid()
self.assertIsInstance(uuid, basestring)
self.assertEqual(len(self.Server._uuids), 999)
uuid2 = self.Server.next_uuid()
self.assertNotEqual(uuid, uuid2)
self.assertEqual(len(self.Server._uuids), 998)
logging.error(
"Please supply one or more URL's of CouchDB instances to update views for."
)
sys.exit(1)
for server_url in server_urls:
try:
server = Server(server_url)
except:
logging.error("Could not connect to CouchDB at %s", server_url)
continue
logging.info("Connected to CouchDB server at %s", server_url)
logging.info("Getting list of databases...")
try:
db_names = server.all_dbs()
except:
logging.error("Could not get list of databases for CouchDB at %s",
server_url)
continue
logging.info("Databases: %s", ", ".join(db_names))
for db_name in db_names:
if db_name.startswith('_'):
logging.info("Skipping database starting with '_': %s", db_name)
continue
logging.info("Database: %s", db_name)
db = server.get_or_create_db(db_name)
for design_doc in db.all_docs(startkey="_design",
#!/usr/bin/env python2
from couchdbkit import Server
s = Server()
print '\n'.join(db for db in s.all_dbs() if db.startswith('test_docs'))
class CouchdbManager(PersistenceManager):
""" This is a couchdb manager for the workspace, it will load from the
couchdb databases"""
def __init__(self, uri):
self._last_seq_ack = 0
model.api.log("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
try:
self.testCouchUrl(uri)
self.__serv = Server(uri = uri)
self._available = True
except:
model.api.log("No route to couchdb server on: %s" % uri)
def isAvailable(self):
return self._available
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchdbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
host, port = None, None
try:
import socket
proto, netloc, _, _, _ = urlsplit(uri)
host, port = splitport(netloc)
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
model.api.log("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
_, netloc, _, _, _ = urlsplit(uri)
host, port = splitport(netloc)
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
@trap_timeout
def getWorkspacesNames(self):
return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs())
def workspaceExists(self, name):
return name in self.getWorkspacesNames()
@trap_timeout
def addWorkspace(self, aWorkspace):
self.__serv.create_db(aWorkspace.lower())
return self.__getDb(aWorkspace)
@trap_timeout
def addDocument(self, aWorkspaceName, documentId, aDocument):
self.incrementSeqNumber(aWorkspaceName)
self.__getDb(aWorkspaceName)[documentId] = aDocument
@trap_timeout
def saveDocument(self, aWorkspaceName, aDocument):
self.incrementSeqNumber(aWorkspaceName)
model.api.log("Saving document in remote workspace %s" % aWorkspaceName)
self.__getDb(aWorkspaceName).save_doc(aDocument, use_uuids = True, force_update = True)
@trap_timeout
def __getDb(self, aWorkspaceName):
aWorkspaceName = aWorkspaceName.lower()
model.api.log("Getting workspace [%s]" % aWorkspaceName)
workspacedb = self.__dbs.get(aWorkspaceName, self.__serv.get_db(aWorkspaceName))
if not self.__dbs.has_key(aWorkspaceName):
model.api.log("Asking couchdb for workspace [%s]" % aWorkspaceName)
self.__dbs[aWorkspaceName] = workspacedb
self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq']
return workspacedb
@trap_timeout
def getDocument(self, aWorkspaceName, documentId):
model.api.log("Getting document for workspace [%s]" % aWorkspaceName)
return self.__getDb(aWorkspaceName).get(documentId)
@trap_timeout
def checkDocument(self, aWorkspaceName, documentName):
return self.__getDb(aWorkspaceName).doc_exist(documentName)
@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
model.api.log("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
model.api.devlog("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
model.api.devlog(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
def getLastChangeSeq(self, workspaceName):
self.mutex.acquire()
seq = self.__seq_nums[workspaceName]
self.mutex.release()
return seq
def setLastChangeSeq(self, workspaceName, seq_num):
self.mutex.acquire()
self.__seq_nums[workspaceName] = seq_num
self.mutex.release()
@trap_timeout
def waitForDBChange(self, db_name, since = 0, timeout = 15000):
""" Be warned this will return after the database has a change, if
there was one before call it will return immediatly with the changes
done"""
changes = []
last_seq = max(self.getLastChangeSeq(db_name), since)
db = self.__getDb(db_name)
with ChangesStream(db, feed="longpoll", since = last_seq, timeout = timeout) as stream:
for change in stream:
if change['seq'] > self.getLastChangeSeq(db_name):
changes.append(change)
last_seq = reduce(lambda x,y: max(y['seq'], x) , changes, self.getLastChangeSeq(db_name))
self.setLastChangeSeq(db_name, last_seq)
return changes
@trap_timeout
def delete_all_dbs(self):
for db in self.__serv.all_dbs():
self.__serv.delete_db(db)
@trap_timeout
def existWorkspace(self, name):
return name in self.__serv.all_dbs()
@trap_timeout
def workspaceDocumentsIterator(self, workspaceName):
return filter(lambda x: not x["id"].startswith("_"), self.__getDb(workspaceName).documents(include_docs=True))
@trap_timeout
def removeWorkspace(self, workspace_name):
return self.__serv.delete_db(workspace_name)
@trap_timeout
def remove(self, workspace, host_id):
self.incrementSeqNumber(workspace)
self.__dbs[workspace].delete_doc(host_id)
@trap_timeout
def compactDatabase(self, aWorkspaceName):
self.__getDb(aWorkspaceName).compact()
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def addViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
for v in vmanager.getAvailableViews():
vmanager.addView(v, workspace)
def getViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
return vmanager.getViews(workspace)
def syncWorkspaceViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
installed_views = vmanager.getViews(workspace)
for v in vmanager.getAvailableViews():
if v not in installed_views:
vmanager.addView(v, workspace)
def incrementSeqNumber(self, workspaceName):
self.mutex.acquire()
self.__seq_nums[workspaceName] += 1
self.mutex.release()
def tearDown(self):
server = Server(uri=CONF.getCouchURI())
if self.dbname in server.all_dbs():
server.delete_db(self.dbname)
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri):
super(CouchDbManager, self).__init__()
getLogger(self).debug(
"Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__serv = NoConectionServer()
self._available = False
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug(
"Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
#@trap_timeout
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
#@trap_timeout
def _delete(self, name):
self.__serv.delete_db(name)
#@trap_timeout
def _loadDbs(self):
conditions = lambda x: not x.startswith("_") and x != 'reports'
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization" % dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
#@trap_timeout
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchDbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
if uri is not None:
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
#getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
#@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri):
super(CouchDbManager, self).__init__()
getLogger(self).debug("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__serv = NoConectionServer()
self._available = False
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug("Setting user,pass %s %s" %
(url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username,
url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
#@trap_timeout
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
#@trap_timeout
def _delete(self, name):
self.__serv.delete_db(name)
#@trap_timeout
def _loadDbs(self):
conditions = lambda x: not x.startswith("_"
) and x not in CONST_BLACKDBS
try:
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization"
% dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
except restkit.errors.RequestError as req_error:
getLogger(self).error(
"Couldn't load databases. "
"The connection to the CouchDB was probably lost. ")
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
def refreshDbs(self):
"""Refresh databases using inherited method. On exception, asume
no databases are available.
"""
try:
return AbstractPersistenceManager.refreshDbs()
except:
return []
#@trap_timeout
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
try:
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
except:
getLogger(self).warn(
"Reports database couldn't be uploaded. You need to be an admin to do it"
)
return self.__uri + "/reports/_design/reports/index.html"
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchDbManager.testCouch(ur):
self.__serv = Server(uri=ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
"""Redirect to the module-level function of the name, which
serves the same purpose and is used by other classes too."""
return test_couch(uri)
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
#@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info(
"workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s"
% (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path,
**kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace,
dst,
mutual=mutual,
continuous=continuous,
create_target=ct)
if mutual:
self.__serv.replicate(dst, src, continuous=continuous, **kwargs)
