class RevCountResolutionTestCase(ConflictResolverTestCase):
"""
This tests resolution that is based on rev_count, given
the time difference between two conflicting versions
is less than configuration.SESSION_LAG_TIME, we can't
rely on timestamps due to server time skew.
So we decide the winning document based on the number
of modifications, assuming the one with the largest
number of modifs should be the current one.
"""
def _runTest(self):
self.s.replicate(self.database_name, 'http://localhost:5984/'+self.repdb_name)
self.s = Server('http://localhost:5984')
self.repdb = self.s.get_or_create_db(self.repdb_name)
self.replicated_doc = self.repdb.get(self.docid)
# increasing the revision log (add 6 more revisions)
for i in range(6):
self.replicated_doc['text'] = 'bigger revision number'
self.replicated_doc['timestamp'] = str(int(round(time.time())))
self.replicated_doc['rev_count'] = str(int(self.replicated_doc['rev_count']) + 1)
self.repdb.save_doc(self.replicated_doc)
# create the conflict, change the same
# text field of the original at the source database.
master_db = self.s.get_or_create_db(self.database_name)
doc = master_db.get(self.docid)
doc['text'] = 'smaller revision number'
doc['timestamp'] = str(int(round(time.time())))
doc['rev_count'] = str(int(doc['rev_count']) + 1)
master_db.save_doc(doc)
self.s.replicate('http://localhost:5984/'+self.repdb_name, self.database_name)
doc = self.db.get(self.docid)
self.assertEqual(doc['text'], 'bigger revision number')
start_time = time.time()
while (self.db.get(self.docid, conflicts=True).has_key('_conflicts')):
pass
end_time = time.time()
print "Time to conflicts clear: %s" % (end_time - start_time)
def runTest(self):
for i in range(10):
self._runTest()
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri):
super(CouchDbManager, self).__init__()
getLogger(self).debug(
"Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__serv = NoConectionServer()
self._available = False
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug(
"Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
#@trap_timeout
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
#@trap_timeout
def _delete(self, name):
self.__serv.delete_db(name)
#@trap_timeout
def _loadDbs(self):
conditions = lambda x: not x.startswith("_") and x != 'reports'
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization" % dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
#@trap_timeout
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchDbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
if uri is not None:
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
#getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
#@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class CouchdbManager(PersistenceManager):
""" This is a couchdb manager for the workspace, it will load from the
couchdb databases"""
def __init__(self, uri):
self._last_seq_ack = 0
model.api.log("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
try:
self.testCouchUrl(uri)
self.__serv = Server(uri = uri)
self._available = True
except:
model.api.log("No route to couchdb server on: %s" % uri)
def isAvailable(self):
return self._available
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchdbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
host, port = None, None
try:
import socket
proto, netloc, _, _, _ = urlsplit(uri)
host, port = splitport(netloc)
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
model.api.log("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
_, netloc, _, _, _ = urlsplit(uri)
host, port = splitport(netloc)
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
@trap_timeout
def getWorkspacesNames(self):
return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs())
def workspaceExists(self, name):
return name in self.getWorkspacesNames()
@trap_timeout
def addWorkspace(self, aWorkspace):
self.__serv.create_db(aWorkspace.lower())
return self.__getDb(aWorkspace)
@trap_timeout
def addDocument(self, aWorkspaceName, documentId, aDocument):
self.incrementSeqNumber(aWorkspaceName)
self.__getDb(aWorkspaceName)[documentId] = aDocument
@trap_timeout
def saveDocument(self, aWorkspaceName, aDocument):
self.incrementSeqNumber(aWorkspaceName)
model.api.log("Saving document in remote workspace %s" % aWorkspaceName)
self.__getDb(aWorkspaceName).save_doc(aDocument, use_uuids = True, force_update = True)
@trap_timeout
def __getDb(self, aWorkspaceName):
aWorkspaceName = aWorkspaceName.lower()
model.api.log("Getting workspace [%s]" % aWorkspaceName)
workspacedb = self.__dbs.get(aWorkspaceName, self.__serv.get_db(aWorkspaceName))
if not self.__dbs.has_key(aWorkspaceName):
model.api.log("Asking couchdb for workspace [%s]" % aWorkspaceName)
self.__dbs[aWorkspaceName] = workspacedb
self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq']
return workspacedb
@trap_timeout
def getDocument(self, aWorkspaceName, documentId):
model.api.log("Getting document for workspace [%s]" % aWorkspaceName)
return self.__getDb(aWorkspaceName).get(documentId)
@trap_timeout
def checkDocument(self, aWorkspaceName, documentName):
return self.__getDb(aWorkspaceName).doc_exist(documentName)
@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
model.api.log("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
model.api.devlog("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
model.api.devlog(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
def getLastChangeSeq(self, workspaceName):
self.mutex.acquire()
seq = self.__seq_nums[workspaceName]
self.mutex.release()
return seq
def setLastChangeSeq(self, workspaceName, seq_num):
self.mutex.acquire()
self.__seq_nums[workspaceName] = seq_num
self.mutex.release()
@trap_timeout
def waitForDBChange(self, db_name, since = 0, timeout = 15000):
""" Be warned this will return after the database has a change, if
there was one before call it will return immediatly with the changes
done"""
changes = []
last_seq = max(self.getLastChangeSeq(db_name), since)
db = self.__getDb(db_name)
with ChangesStream(db, feed="longpoll", since = last_seq, timeout = timeout) as stream:
for change in stream:
if change['seq'] > self.getLastChangeSeq(db_name):
changes.append(change)
last_seq = reduce(lambda x,y: max(y['seq'], x) , changes, self.getLastChangeSeq(db_name))
self.setLastChangeSeq(db_name, last_seq)
return changes
@trap_timeout
def delete_all_dbs(self):
for db in self.__serv.all_dbs():
self.__serv.delete_db(db)
@trap_timeout
def existWorkspace(self, name):
return name in self.__serv.all_dbs()
@trap_timeout
def workspaceDocumentsIterator(self, workspaceName):
return filter(lambda x: not x["id"].startswith("_"), self.__getDb(workspaceName).documents(include_docs=True))
@trap_timeout
def removeWorkspace(self, workspace_name):
return self.__serv.delete_db(workspace_name)
@trap_timeout
def remove(self, workspace, host_id):
self.incrementSeqNumber(workspace)
self.__dbs[workspace].delete_doc(host_id)
@trap_timeout
def compactDatabase(self, aWorkspaceName):
self.__getDb(aWorkspaceName).compact()
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def addViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
for v in vmanager.getAvailableViews():
vmanager.addView(v, workspace)
def getViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
return vmanager.getViews(workspace)
def syncWorkspaceViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
installed_views = vmanager.getViews(workspace)
for v in vmanager.getAvailableViews():
if v not in installed_views:
vmanager.addView(v, workspace)
def incrementSeqNumber(self, workspaceName):
self.mutex.acquire()
self.__seq_nums[workspaceName] += 1
self.mutex.release()
class CouchdbManager(PersistenceManager):
""" This is a couchdb manager for the workspace, it will load from the
couchdb databases"""
def __init__(self, uri):
self._last_seq_ack = 0
model.api.log("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
try:
self.testCouchUrl(uri)
url = urlparse(uri)
print("Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
#print dir(self.__serv)
self.__serv.resource_class.credentials = (url.username,
url.password)
self._available = True
except:
model.api.log("No route to couchdb server on: %s" % uri)
print(traceback.format_exc())
def isAvailable(self):
return self._available
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchdbManager.testCouch(ur):
self.__serv = Server(uri=ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
model.api.log("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
@trap_timeout
def getWorkspacesNames(self):
return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs())
def workspaceExists(self, name):
return name in self.getWorkspacesNames()
@trap_timeout
def addWorkspace(self, aWorkspace):
self.__serv.create_db(aWorkspace.lower())
return self.__getDb(aWorkspace)
@trap_timeout
def addDocument(self, aWorkspaceName, documentId, aDocument):
self.incrementSeqNumber(aWorkspaceName)
self.__getDb(aWorkspaceName)[documentId] = aDocument
@trap_timeout
def saveDocument(self, aWorkspaceName, aDocument):
self.incrementSeqNumber(aWorkspaceName)
model.api.log("Saving document in remote workspace %s" %
aWorkspaceName)
self.__getDb(aWorkspaceName).save_doc(aDocument,
use_uuids=True,
force_update=True)
@trap_timeout
def __getDb(self, aWorkspaceName):
aWorkspaceName = aWorkspaceName.lower()
model.api.log("Getting workspace [%s]" % aWorkspaceName)
workspacedb = self.__dbs.get(aWorkspaceName,
self.__serv.get_db(aWorkspaceName))
if not self.__dbs.has_key(aWorkspaceName):
model.api.log("Asking couchdb for workspace [%s]" % aWorkspaceName)
self.__dbs[aWorkspaceName] = workspacedb
self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq']
return workspacedb
@trap_timeout
def getDocument(self, aWorkspaceName, documentId):
model.api.log("Getting document for workspace [%s]" % aWorkspaceName)
return self.__getDb(aWorkspaceName).get(documentId)
@trap_timeout
def checkDocument(self, aWorkspaceName, documentName):
return self.__getDb(aWorkspaceName).doc_exist(documentName)
@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
model.api.log("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
model.api.devlog(
"workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s"
% (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path,
**kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
model.api.devlog(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace,
dst,
mutual=mutual,
continuous=continuous,
create_target=ct)
if mutual:
self.__serv.replicate(dst, src, continuous=continuous, **kwargs)
def getLastChangeSeq(self, workspaceName):
self.mutex.acquire()
seq = self.__seq_nums[workspaceName]
self.mutex.release()
return seq
def setLastChangeSeq(self, workspaceName, seq_num):
self.mutex.acquire()
self.__seq_nums[workspaceName] = seq_num
self.mutex.release()
@trap_timeout
def waitForDBChange(self, db_name, since=0, timeout=15000):
""" Be warned this will return after the database has a change, if
there was one before call it will return immediatly with the changes
done"""
changes = []
last_seq = max(self.getLastChangeSeq(db_name), since)
db = self.__getDb(db_name)
with ChangesStream(db,
feed="longpoll",
since=last_seq,
timeout=timeout) as stream:
for change in stream:
if change['seq'] > self.getLastChangeSeq(db_name):
changes.append(change)
last_seq = reduce(lambda x, y: max(y['seq'], x), changes,
self.getLastChangeSeq(db_name))
self.setLastChangeSeq(db_name, last_seq)
return changes
@trap_timeout
def delete_all_dbs(self):
for db in self.__serv.all_dbs():
self.__serv.delete_db(db)
@trap_timeout
def existWorkspace(self, name):
return name in self.__serv.all_dbs()
@trap_timeout
def workspaceDocumentsIterator(self, workspaceName):
return filter(lambda x: not x["id"].startswith("_"),
self.__getDb(workspaceName).documents(include_docs=True))
@trap_timeout
def removeWorkspace(self, workspace_name):
return self.__serv.delete_db(workspace_name)
@trap_timeout
def remove(self, workspace, host_id):
self.incrementSeqNumber(workspace)
self.__dbs[workspace].delete_doc(host_id)
@trap_timeout
def compactDatabase(self, aWorkspaceName):
self.__getDb(aWorkspaceName).compact()
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def addViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
for v in vmanager.getAvailableViews():
vmanager.addView(v, workspace)
def getViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
return vmanager.getViews(workspace)
def syncWorkspaceViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self.__getDb(workspaceName)
installed_views = vmanager.getViews(workspace)
for v in vmanager.getAvailableViews():
if v not in installed_views:
vmanager.addView(v, workspace)
def incrementSeqNumber(self, workspaceName):
self.mutex.acquire()
self.__seq_nums[workspaceName] += 1
self.mutex.release()
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri):
super(CouchDbManager, self).__init__()
getLogger(self).debug(
"Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__serv = NoConectionServer()
self._available = False
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug(
"Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
#@trap_timeout
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
#@trap_timeout
def _delete(self, name):
self.__serv.delete_db(name)
#@trap_timeout
def _loadDbs(self):
conditions = lambda x: not x.startswith("_") and x != 'reports'
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization" % dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
#@trap_timeout
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchDbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
if uri is not None:
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
#getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
#@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class CouchdbManager(PersistenceManager):
""" This is a couchdb manager for the workspace, it will load from the
couchdb databases"""
def __init__(self, uri):
self._last_seq_ack = 0
getLogger(self).debug("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__dbs = {}
self.__seq_nums = {}
self.__serv = NoConectionServer()
self.mutex = threading.Lock()
self._available = False
#setting the doc types to load from couch
def get_types(subclasses):
if len(subclasses):
head = subclasses[0]
tail = []
if len(subclasses[1:]):
tail = subclasses[1:]
return get_types(head.__subclasses__()) + [head.class_signature] + get_types(tail)
return []
self._model_object_types = get_types([ModelObject])
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug("Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
#print dir(self.__serv)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
def isAvailable(self):
return self._available
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchdbManager.testCouch(ur):
self.__serv = Server(uri = ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
if uri is not None:
host, port = None, None
try:
import socket
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
port = port if port else socket.getservbyname(proto)
s = socket.socket()
s.settimeout(1)
s.connect((host, int(port)))
except:
return False
getLogger(CouchdbManager).info("Connecting Couch to: %s:%s" % (host, port))
return True
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
@trap_timeout
def getWorkspacesNames(self):
return filter(lambda x: not x.startswith("_"), self.__serv.all_dbs())
def workspaceExists(self, name):
return name in self.getWorkspacesNames()
@trap_timeout
def addWorkspace(self, aWorkspace):
self.__serv.create_db(aWorkspace.lower())
return self._getDb(aWorkspace)
@trap_timeout
def addDocument(self, aWorkspaceName, documentId, aDocument):
self._getDb(aWorkspaceName)
self.incrementSeqNumber(aWorkspaceName)
self._getDb(aWorkspaceName)[documentId] = aDocument
@trap_timeout
def saveDocument(self, aWorkspaceName, aDocument):
self.incrementSeqNumber(aWorkspaceName)
getLogger(self).debug("Saving document in remote workspace %s" % aWorkspaceName)
return self._getDb(aWorkspaceName).save_doc(aDocument, use_uuids = True, force_update = True)
def _getDb(self, aWorkspaceName):
if not self.__dbs.has_key(aWorkspaceName):
self.__getDb(aWorkspaceName)
return self.__dbs.get(aWorkspaceName, None)
@trap_timeout
def __getDb(self, aWorkspaceName):
aWorkspaceName = aWorkspaceName.lower()
getLogger(self).debug("Getting workspace [%s]" % aWorkspaceName)
workspacedb = self.__dbs.get(aWorkspaceName, self.__serv.get_db(aWorkspaceName))
if not self.__dbs.has_key(aWorkspaceName):
getLogger(self).debug("Asking couchdb for workspace [%s]" % aWorkspaceName)
self.__dbs[aWorkspaceName] = workspacedb
self.__seq_nums[aWorkspaceName] = workspacedb.info()['update_seq']
return workspacedb
@trap_timeout
def getDocument(self, aWorkspaceName, documentId):
getLogger(self).debug("Getting document for workspace [%s]" % aWorkspaceName)
return self._getDb(aWorkspaceName).get(documentId)
@trap_timeout
def getDeletedDocument(self, aWorkspaceName, documentId, documentRev):
return self._getDb(aWorkspaceName).get(documentId, rev=documentRev)
@trap_timeout
def checkDocument(self, aWorkspaceName, documentName):
return self._getDb(aWorkspaceName).doc_exist(documentName)
@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
def getLastChangeSeq(self, workspaceName):
self.mutex.acquire()
seq = self.__seq_nums[workspaceName]
self.mutex.release()
return seq
def setLastChangeSeq(self, workspaceName, seq_num):
self.mutex.acquire()
self.__seq_nums[workspaceName] = seq_num
self.mutex.release()
@trap_timeout
def waitForDBChange(self, db_name, since = 0, timeout = 15000):
""" Be warned this will return after the database has a change, if
there was one before call it will return immediatly with the changes
done"""
changes = []
last_seq = max(self.getLastChangeSeq(db_name), since)
db = self._getDb(db_name)
with ChangesStream(db, feed="longpoll", since=last_seq, timeout=timeout) as stream:
for change in stream:
if change['seq'] > self.getLastChangeSeq(db_name):
self.setLastChangeSeq(db_name, change['seq'])
if not change['id'].startswith('_design'):
#fake doc type for deleted objects
doc = {'type': 'unknown', '_deleted': 'False', '_rev':[0]}
if not change.get('deleted'):
doc = self.getDocument(db_name, change['id'])
changes.append(change_factory.create(doc))
if len(changes):
getLogger(self).debug("Changes from another instance")
return changes
@trap_timeout
def delete_all_dbs(self):
for db in self.__serv.all_dbs():
self.__serv.delete_db(db)
@trap_timeout
def existWorkspace(self, name):
return name in self.__serv.all_dbs()
@trap_timeout
def workspaceDocumentsIterator(self, workspaceName):
return filter(self.filterConditions, self._getDb(workspaceName).documents(include_docs=True))
def filterConditions(self, doc):
ret = True
ret = ret and not doc["id"].startswith("_")
ret = ret and doc['doc']["type"] in self._model_object_types
return ret
@trap_timeout
def removeWorkspace(self, workspace_name):
return self.__serv.delete_db(workspace_name)
@trap_timeout
def remove(self, workspace, host_id):
self.incrementSeqNumber(workspace)
self.__dbs[workspace].delete_doc(host_id)
@trap_timeout
def compactDatabase(self, aWorkspaceName):
self._getDb(aWorkspaceName).compact()
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
return self.__uri + "/reports/_design/reports/index.html"
def addViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self._getDb(workspaceName)
for v in vmanager.getAvailableViews():
vmanager.addView(v, workspace)
def getViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self._getDb(workspaceName)
return vmanager.getViews(workspace)
def syncWorkspaceViews(self, workspaceName):
vmanager = ViewsManager()
workspace = self._getDb(workspaceName)
installed_views = vmanager.getViews(workspace)
for v in vmanager.getAvailableViews():
if v not in installed_views:
vmanager.addView(v, workspace)
def incrementSeqNumber(self, workspaceName):
self.mutex.acquire()
if not self.__seq_nums.has_key(workspaceName):
self.__seq_nums[workspaceName] = 0
self.__seq_nums[workspaceName] += 1
self.mutex.release()
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri, couch_exception_callback):
super(CouchDbManager, self).__init__()
getLogger(self).debug(
"Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self._available = False
self.couch_exception_callback = couch_exception_callback
test_couch_thread = threading.Thread(target=self.continuosly_check_connection)
test_couch_thread.daemon = True
test_couch_thread.start()
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug(
"Setting user,pass %s %s" % (url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username, url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
def continuosly_check_connection(self):
"""Intended to use on a separate thread. Call module-level
function testCouch every second to see if response to the server_uri
of the DB is still 200. Call the exception_callback if we can't access
the server three times in a row.
"""
tolerance = 0
server_uri = self.__uri
while True:
time.sleep(1)
test_was_successful = test_couch(server_uri)
if test_was_successful:
tolerance = 0
else:
tolerance += 1
if tolerance == 3:
self.couch_exception_callback()
return False # kill the thread if something went wrong
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
def _delete(self, name):
self.__serv.delete_db(name)
def _loadDbs(self):
def conditions(database):
begins_with_underscore = database.startswith("_")
is_blacklisted = database in CONST_BLACKDBS
return not begins_with_underscore and not is_blacklisted
try:
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization" % dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
except restkit.errors.RequestError as req_error:
getLogger(self).error("Couldn't load databases. "
"The connection to the CouchDB was probably lost. ")
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
def refreshDbs(self):
"""Refresh databases using inherited method. On exception, asume
no databases are available.
"""
try:
return AbstractPersistenceManager.refreshDbs()
except:
return []
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
try:
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
except:
getLogger(self).warn(
"Reports database couldn't be uploaded. You need to be an admin to do it")
return self.__uri + "/reports/_design/reports/index.html"
@staticmethod
def testCouch(uri):
"""Redirect to the module-level function of the name, which
serves the same purpose and is used by other classes too."""
return test_couch(uri)
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info("workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s" % (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path, **kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace, dst, mutual = mutual, continuous = continuous, create_target = ct)
if mutual:
self.__serv.replicate(dst, src, continuous = continuous, **kwargs)
class ConnectionManager(object):
"""
This class takes care of starting the wrapper SSH tunnel connections to allow
communicating with the remote CouchDB nodes, and is responsible for restarting
replication connections for nodes that go back into online state as reported
by the NodeProbe.
"""
def __init__(self, hosts, database_name="session_store"):
self.logger = configuration.get_logger(logging_instance=connlogging, system_name="connmanager")
configuration.info(self.logger)
self.database_name = database_name
self.hosts = hosts
self.online = []
self.ip = self.get_ip_address()
self.logger.info("= ConnectionManager instantiated =")
self.logger.info("My IP -> %s" % self.ip)
self.logger.info("Targets: %s" % self.hosts)
# Create an instance to the local server
self.dbserver = Server()
self.node_probe = None # reference for the NodeProbe object
def manage(self):
"""This encapsulates one monitoring run"""
# clear latest run results
# (if not our lists get inifnitely populated)
self.online = []
self.offline = []
# start the monitor run
self.logger.info("Monitor run for %s" % self.hosts)
self.node_probe = NodeProbe(self.hosts, self.online, self.offline)
self.node_probe.survey()
self.node_probe.wait_finish()
print "online: %s" % self.online
print "offline: %s" % self.offline
for i in self.online:
self.logger.info("%s online. Restarting connection." % i[1])
self.restartConnection(i[1])
def get_ip_address(self):
return configuration.MY_IP
def restartConnection(self, target_uri):
"""
This wrapper may look redundant, but it is here to
remind us that eventually SSH tunneling will be handled by
'self.startTunnel' and will be called before 'self.continuousReplication'.
This will mandate the translation of the real ip addresses to localhost
and respective port to make the SSH tunneling transparent to users of
the ConnectionManager. So once it is finished, target nodes list will be fetched
from the configuration CouchDB db and translated to tunnel invocations
and localhost replication connections.
"""
self.logger.info("Restarting conn. for %s" % target_uri)
self.continuousReplication(target_uri, self.database_name)
def manageForever(self, interval=30):
while True:
self.manage()
time.sleep(interval)
def startTunnel(self, local_port, local_host, target_port, target_host):
"""
If not already started, start a new autoSSH process to
keep the connection to the target.
If the autossh process is already there, leave it since autossh
takes care of maintaining the connection.
Return the 'http://localhost:900x' equivalent for the real ip and port.
e.g: 'http://79.143.23.119:5984' --> 'http://localhost:9001'
This enables transparent restart of the CouchDB plain text HTTP replication
connections.
"""
def continuousReplication(self, target_uri, database_name):
"""
Stop continuous replication to target_uri if exists.
then, start it fresh.
< rnewson> sivang: you can cancel a replication with "cancel":true but
they are not automatically restarted if they crash. However:
According to rnewson starting connection reuses a previous connection if it existed,
we don't really need to do anything.
Just - Start the continuous replication again for every node that came back online,
and forget about it! yes, it is THAT easy.
This is after all, CouchDB. Time to relax.
"""
target_uri_db = "%s/%s" % (target_uri, database_name)
self.logger.info("Start Cont. rpct. : %s" % target_uri_db)
# direction of replication is changed to pull replication instead
# of push, as recommended by the CouchDB wiki for better performance.
self.dbserver.replicate(source=self.database_name, target=target_uri_db, continuous=True)
class CouchDbManager(AbstractPersistenceManager):
"""
This is a couchdb manager for the workspace,
it will load from the couchdb databases
"""
def __init__(self, uri):
super(CouchDbManager, self).__init__()
getLogger(self).debug("Initializing CouchDBManager for url [%s]" % uri)
self._lostConnection = False
self.__uri = uri
self.__serv = NoConectionServer()
self._available = False
try:
if uri is not None:
self.testCouchUrl(uri)
url = urlparse(uri)
getLogger(self).debug("Setting user,pass %s %s" %
(url.username, url.password))
self.__serv = Server(uri=uri)
self.__serv.resource_class.credentials = (url.username,
url.password)
self._available = True
self.pushReports()
self._loadDbs()
except:
getLogger(self).warn("No route to couchdb server on: %s" % uri)
getLogger(self).debug(traceback.format_exc())
#@trap_timeout
def _create(self, name):
db = self.__serv.create_db(name.lower())
return CouchDbConnector(db)
#@trap_timeout
def _delete(self, name):
self.__serv.delete_db(name)
#@trap_timeout
def _loadDbs(self):
conditions = lambda x: not x.startswith("_"
) and x not in CONST_BLACKDBS
try:
for dbname in filter(conditions, self.__serv.all_dbs()):
if dbname not in self.dbs.keys():
getLogger(self).debug(
"Asking for dbname[%s], registering for lazy initialization"
% dbname)
self.dbs[dbname] = lambda x: self._loadDb(x)
except restkit.errors.RequestError as req_error:
getLogger(self).error(
"Couldn't load databases. "
"The connection to the CouchDB was probably lost. ")
def _loadDb(self, dbname):
db = self.__serv.get_db(dbname)
seq = db.info()['update_seq']
self.dbs[dbname] = CouchDbConnector(db, seq_num=seq)
return self.dbs[dbname]
def refreshDbs(self):
"""Refresh databases using inherited method. On exception, asume
no databases are available.
"""
try:
return AbstractPersistenceManager.refreshDbs()
except:
return []
#@trap_timeout
def pushReports(self):
vmanager = ViewsManager()
reports = os.path.join(os.getcwd(), "views", "reports")
try:
workspace = self.__serv.get_or_create_db("reports")
vmanager.addView(reports, workspace)
except:
getLogger(self).warn(
"Reports database couldn't be uploaded. You need to be an admin to do it"
)
return self.__uri + "/reports/_design/reports/index.html"
def lostConnectionResolv(self):
self._lostConnection = True
self.__dbs.clear()
self.__serv = NoConectionServer()
def reconnect(self):
ret_val = False
ur = self.__uri
if CouchDbManager.testCouch(ur):
self.__serv = Server(uri=ur)
self.__dbs.clear()
self._lostConnection = False
ret_val = True
return ret_val
@staticmethod
def testCouch(uri):
"""Redirect to the module-level function of the name, which
serves the same purpose and is used by other classes too."""
return test_couch(uri)
def testCouchUrl(self, uri):
if uri is not None:
url = urlparse(uri)
proto = url.scheme
host = url.hostname
port = url.port
self.test(host, int(port))
def test(self, address, port):
import socket
s = socket.socket()
s.settimeout(1)
s.connect((address, port))
#@trap_timeout
def replicate(self, workspace, *targets_dbs, **kwargs):
getLogger(self).debug("Targets to replicate %s" % str(targets_dbs))
for target_db in targets_dbs:
src_db_path = "/".join([self.__uri, workspace])
dst_db_path = "/".join([target_db, workspace])
try:
getLogger(self).info(
"workspace: %s, src_db_path: %s, dst_db_path: %s, **kwargs: %s"
% (workspace, src_db_path, dst_db_path, kwargs))
self.__peerReplication(workspace, src_db_path, dst_db_path,
**kwargs)
except ResourceNotFound as e:
raise e
except Exception as e:
getLogger(self).error(e)
raise
def __peerReplication(self, workspace, src, dst, **kwargs):
mutual = kwargs.get("mutual", True)
continuous = kwargs.get("continuous", True)
ct = kwargs.get("create_target", True)
self.__serv.replicate(workspace,
dst,
mutual=mutual,
continuous=continuous,
create_target=ct)
if mutual:
self.__serv.replicate(dst, src, continuous=continuous, **kwargs)
