def __init__(self, config):
# configuration values:
# 'uri' attribute (URL of the REST server and resource name)
# in case of CouchDB, the resource name is the database name
# http://servername:port/databaseName
self.config = config
# the class currently relies only on 1 REST server possibility - the
# CouchDB server. as explained above, .database will be replaced by
# .connection if both a generic REST server as well as CouchDB are to
# be talked to
split = self.config.uri.rfind('/')
dbName = self.config.uri[split + 1:] # get last item of URI - database name
url = self.config.uri[:split]
# as opposed to CouchSink, here it's assumed the resource (the database name)
# does exist, fail here otherwise
# this check / rest of the constructed may be revised for
# general REST server
server = CouchServer(url)
databases = server.listDatabases()
if dbName not in databases:
raise Exception("%s: REST URI: %s, %s does not exist." %
(self.__class__.__name__, self.config.uri, dbName))
self._database = Database(dbName, url)
logging.debug("%s initialized." % self.__class__.__name__)
class WMLoggingTest(unittest.TestCase):
def setUp(self):
# Make an instance of the server
self.server = CouchServer(os.getenv("COUCHURL", 'http://*****:*****@localhost:5984'))
testname = self.id().split('.')[-1]
# Create a database, drop an existing one first
self.dbname = 'cmscouch_unittest_%s' % testname.lower()
if self.dbname in self.server.listDatabases():
self.server.deleteDatabase(self.dbname)
self.server.createDatabase(self.dbname)
self.db = self.server.connectDatabase(self.dbname)
def tearDown(self):
# This used to test self._exc_info to only run on success. Broke in 2.7. Removed.
self.server.deleteDatabase(self.dbname)
def testLog(self):
"""
Write ten log messages to the database at three different levels
"""
my_logger = logging.getLogger('MyLogger')
my_logger.setLevel(logging.DEBUG)
handler = CouchHandler(self.server.url, self.dbname)
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
my_logger.addHandler(handler)
for _ in range(10):
my_logger.debug('This is probably all noise.')
my_logger.info('Jackdaws love my big sphinx of quartz.')
my_logger.error('HOLLY CRAP!')
logs = self.db.allDocs()['rows']
self.assertEqual(30, len(logs))
def __init__(self, config):
# configuration values:
# 'uri' attribute (URL of the REST server and resource name)
# in case of CouchDB, the resource name is the database name
# http://servername:port/databaseName
self.config = config
logging.info("Instantiating ...")
# the class currently relies only on 1 REST server possibility - the
# CouchDB server. as explained above, .database will be replaced by
# .connection if both a generic REST server as well as CouchDB are to
# be talked to
split = self.config.uri.rfind('/')
dbName = self.config.uri[split +
1:] # get last item of URI - database name
url = self.config.uri[:split]
# as opposed to CouchSink, here it's assumed the resource (the database name)
# does exist, fail here otherwise
# this check / rest of the constructed may be revised for
# general REST server
server = CouchServer(url)
databases = server.listDatabases()
# there needs to be this database created upfront and also
# couchapp associated with it installed, if it's there, fail
if dbName not in databases:
raise Exception("REST URI: %s (DB name: %s) does not exist." %
(self.config.uri, dbName))
self._database = Database(dbName, url)
logging.info("Initialized.")
class DQMCouchAPI(WMObject, WMConnectionBase):
"""
Update the harvesting status of a dataset in CouchDB
"""
def __init__(self, config, couchDbName = None, couchurl = None):
WMObject.__init__(self, config)
WMConnectionBase.__init__(self, "WMCore.WMBS")
self.designDoc = "HarvestingDatasets"
if couchDbName == None:
self.dbname = getattr(self.config.HarvestingScheduler, "couchDBName",
"dqm_default")
else:
self.dbname = couchDbName
if couchurl is not None:
self.couchurl = couchurl
elif getattr(self.config.HarvestingScheduler, "couchurl",
None) is not None:
self.couchurl = self.config.HarvestingScheduler.couchurl
else:
self.couchurl = self.config.JobStateMachine.couchurl
try:
self.couchdb = CouchServer(self.couchurl)
if self.dbname not in self.couchdb.listDatabases():
self.createDatabase()
self.database = self.couchdb.connectDatabase(self.dbname,
size=_LIMIT)
except Exception, ex:
logging.error("Error connecting to couch: %s" % str(ex))
self.database = None
return
class CouchAppTestHarness:
"""
Test Harness for installing a couch database instance with several couchapps
in a unittest.setUp and wiping it out in a unittest.tearDown
"""
def __init__(self, dbName, couchUrl = None):
self.couchUrl = os.environ.get("COUCHURL", couchUrl)
self.dbName = dbName
if self.couchUrl == None:
msg = "COUCHRURL env var not set..."
raise RuntimeError, msg
self.couchServer = CouchServer(self.couchUrl)
self.couchappConfig = Config()
def create(self):
"""create couch db instance"""
if self.dbName in self.couchServer.listDatabases():
self.drop()
self.couchServer.createDatabase(self.dbName)
def drop(self):
"""blow away the couch db instance"""
self.couchServer.deleteDatabase(self.dbName)
def pushCouchapps(self, *couchappdirs):
"""
push a list of couchapps to the database
"""
for couchappdir in couchappdirs:
couchapppush(self.couchappConfig, couchappdir, "%s/%s" % (self.couchUrl, urllib.quote_plus(self.dbName)))
class WMLoggingTest(unittest.TestCase):
def setUp(self):
# Make an instance of the server
self.server = CouchServer(os.getenv("COUCHURL", 'http://*****:*****@localhost:5984'))
testname = self.id().split('.')[-1]
# Create a database, drop an existing one first
self.dbname = 'cmscouch_unittest_%s' % testname.lower()
if self.dbname in self.server.listDatabases():
self.server.deleteDatabase(self.dbname)
self.server.createDatabase(self.dbname)
self.db = self.server.connectDatabase(self.dbname)
def tearDown(self):
# This used to test self._exc_info to only run on success. Broke in 2.7. Removed.
self.server.deleteDatabase(self.dbname)
def testLog(self):
"""
Write ten log messages to the database at three different levels
"""
my_logger = logging.getLogger('MyLogger')
my_logger.setLevel(logging.DEBUG)
handler = CouchHandler(self.server.url, self.dbname)
formatter = logging.Formatter('%(message)s')
handler.setFormatter(formatter)
my_logger.addHandler(handler)
for _ in range(10):
my_logger.debug('This is probably all noise.')
my_logger.info('Jackdaws love my big sphinx of quartz.')
my_logger.error('HOLLY CRAP!')
logs = self.db.allDocs()['rows']
self.assertEqual(30, len(logs))
def __init__(self, config):
self.config = config
# test if the configured database does not exist, create it
server = CouchServer(self.config.url)
databases = server.listDatabases()
if self.config.database not in databases:
server.createDatabase(self.config.database)
self.database = Database(self.config.database, self.config.url)
logging.debug("%s initialized." % self.__class__.__name__)
def setUp(self):
couch = CouchServer(dburl=self.DB_URL)
if self.DB_NAME in couch.listDatabases():
couch.deleteDatabase(self.DB_NAME)
cdb = couch.connectDatabase(self.DB_NAME)
#for dq_t in test_data.demo_data:
# cdb.queue(dq_t)
cdb.commit()
self.db = Database(dbname=self.DB_NAME)
def _getDbConnection(couchUrl, dbName):
"""
Check if the database exists, create if not.
"""
couchServer = CouchServer(couchUrl)
if not dbName in couchServer.listDatabases():
logging.info("Database '%s' does not exits, creating it." % dbName)
db = couchServer.createDatabase(dbName) # returns Database
else:
logging.debug("Database '%s' exists." % dbName)
db = Database(dbName, couchUrl)
return couchServer, db
def _getDbConnection(couchUrl, dbName):
"""
Check if the database exists, create if not.
"""
couchServer = CouchServer(couchUrl)
if not dbName in couchServer.listDatabases():
logging.info("Database '%s' does not exits, creating it." % dbName)
db = couchServer.createDatabase(dbName) # returns Database
else:
logging.debug("Database '%s' exists." % dbName)
db = Database(dbName, couchUrl)
return couchServer, db
def __init__(self, config):
self.config = config
logging.info("Instantiating ...")
# test if the configured database does not exist, create it
server = CouchServer(self.config.url)
databases = server.listDatabases()
if self.config.database not in databases:
logging.warn("'%s' database does not exist on %s, creating it ..." %
(self.config.database, self.config.url))
server.createDatabase(self.config.database)
logging.warn("Created.")
logging.info("'%s' database exists on %s" % (self.config.database, self.config.url))
self.database = Database(self.config.database, self.config.url)
logging.info("Initialized.")
def __init__(self, config):
self.config = config
logging.info("Instantiating ...")
# test if the configured database does not exist, create it
server = CouchServer(self.config.url)
databases = server.listDatabases()
if self.config.database not in databases:
logging.warn(
"'%s' database does not exist on %s, creating it ..." %
(self.config.database, self.config.url))
server.createDatabase(self.config.database)
logging.warn("Created.")
logging.info("'%s' database exists on %s" %
(self.config.database, self.config.url))
self.database = Database(self.config.database, self.config.url)
logging.info("Initialized.")
class CouchAppTestHarness:
"""
Test Harness for installing a couch database instance with several couchapps
in a unittest.setUp and wiping it out in a unittest.tearDown
"""
def __init__(self, dbName, couchUrl=None):
self.couchUrl = os.environ.get("COUCHURL", couchUrl)
self.dbName = dbName
if self.couchUrl == None:
msg = "COUCHRURL env var not set..."
raise RuntimeError(msg)
if self.couchUrl.endswith('/'):
raise RuntimeError("COUCHURL env var shouldn't end with /")
self.couchServer = CouchServer(self.couchUrl)
self.couchappConfig = Config()
def create(self, dropExistingDb=True):
"""create couch db instance"""
#import pdb
#pdb.set_trace()
if self.dbName in self.couchServer.listDatabases():
if not dropExistingDb:
return
self.drop()
self.couchServer.createDatabase(self.dbName)
def drop(self):
"""blow away the couch db instance"""
self.couchServer.deleteDatabase(self.dbName)
def pushCouchapps(self, *couchappdirs):
"""
push a list of couchapps to the database
"""
for couchappdir in couchappdirs:
couchapppush(
self.couchappConfig, couchappdir,
"%s/%s" % (self.couchUrl, urllib.quote_plus(self.dbName)))
def _getDbConnection(couchUrl, dbName):
"""
Check if the database exists, create if not.
"""
couchServer = CouchServer(couchUrl)
if not dbName in couchServer.listDatabases():
logging.info("Database '%s' does not exits, creating it." % dbName)
db = couchServer.createDatabase(dbName)
else:
logging.debug("Database '%s' exists." % dbName)
db = Database(dbName, couchUrl)
couchapps = "../../../src/couchapp"
stat_couchapp = "%s/stat" % couchapps
harness = CouchAppTestHarness(dbName, couchUrl)
harness.create()
harness.pushCouchapps(stat_couchapp)
return couchServer, db
def _getDbConnection(couchUrl, dbName):
"""
Check if the database exists, create if not.
"""
couchServer = CouchServer(couchUrl)
if not dbName in couchServer.listDatabases():
logging.info("Database '%s' does not exits, creating it." % dbName)
db = couchServer.createDatabase(dbName)
else:
logging.debug("Database '%s' exists." % dbName)
db = Database(dbName, couchUrl)
couchapps = "../../../src/couchapp"
stat_couchapp = "%s/stat" % couchapps
harness = CouchAppTestHarness(dbName, couchUrl)
harness.create()
harness.pushCouchapps(stat_couchapp)
return couchServer, db
def setUp(self):
dbname = 'dqis_%s' % self.id().split('.')[-1].lower()
couch = CouchServer(dburl='admin:password@localhost:5984')
if dbname not in couch.listDatabases():
couch.createDatabase(dbname)
couch.connectDatabase(dbname)
self.db = Database(dbname, couch.url)
#insert 5 known records
self.db.queue({
"_id": "1234-1-f28ca5a7388b8f6ade71b43c4f4b7669",
"bfield": 4000,
"map": {
"dt_shift_online": True,
"hlt_shift_online": True,
"_meta": {
"timestamp": "2010-04-14 18:04:58.087723",
"user": "******"
}
},
"run": 1234,
"map_history": [
],
"dataset": [
"Cosmics",
"Commissioning09-MuAlStandAloneCosmics-v2",
"ALCARECO"
],
"lumi": 1
})
self.db.queue({
"_id": "1234-2-f28ca5a7388b8f6ade71b43c4f4b7669",
"bfield": 4000,
"map": {
"dt_shift_online": True,
"hlt_shift_online": False,
"_meta": {
"timestamp": "2010-04-14 18:04:58.087723",
"user": "******"
}
},
"run": 1234,
"map_history": [
],
"dataset": [
"Cosmics",
"Commissioning09-MuAlStandAloneCosmics-v2",
"ALCARECO"
],
"lumi": 2
})
self.db.queue({
"_id": "1234-3-f28ca5a7388b8f6ade71b43c4f4b7669",
"bfield": 2000,
"map": {
"dt_shift_online": True,
"hlt_shift_online": True,
"_meta": {
"timestamp": "2010-04-14 18:04:58.087723",
"user": "******"
}
},
"run": 1234,
"map_history": [
],
"dataset": [
"Cosmics",
"Commissioning09-MuAlStandAloneCosmics-v2",
"ALCARECO"
],
"lumi": 3
})
self.db.queue({
"_id": "5678-2-f28ca5a7388b8f6ade71b43c4f4b7669",
"bfield": 2000,
"map": {
"dt_shift_online": False,
"hlt_shift_online": True,
"_meta": {
"timestamp": "2010-04-14 18:04:58.087723",
"user": "******"
}
},
"run": 5678,
"map_history": [
],
"dataset": [
"Cosmics",
"Commissioning09-MuAlStandAloneCosmics-v2",
"ALCARECO"
],
"lumi": 2
})
self.db.queue({
"_id": "5678-1-f28ca5a7388b8f6ade71b43c4f4b7669",
"bfield": 4000,
"map": {
"dt_shift_online": True,
"hlt_shift_online": True,
"_meta": {
"timestamp": "2010-04-14 18:04:58.087723",
"user": "******"
}
},
"run": 5678,
"map_history": [
],
"dataset": [
"Cosmics",
"Commissioning09-MuAlStandAloneCosmics-v2",
"ALCARECO"
],
"lumi": 1
})
self.db.commit()
cmd = "couchapp push ../../../../src/couchapp/dqis/. http://admin:password@localhost:5984/%s" % dbname
commands.getstatusoutput(cmd)
class LogDBBackend(object):
"""
Represents persistent storage for LogDB
"""
def __init__(self, db_url, db_name, identifier, thread_name, **kwds):
self.db_url = db_url
self.server = CouchServer(db_url)
self.db_name = db_name
self.dbid = identifier
self.thread_name = thread_name
self.agent = kwds.get('agent', 0)
self.db = self.server.connectDatabase(db_name, create=False)
self.design = 'LogDB' # name of design document
self.view = 'requests' # name of view to look-up requests
self.tsview = 'tstamp' # name of tsview to look-up requests
self.threadview = 'logByRequestAndThread'
self.requestview = 'logByRequest'
def deleteDatabase(self):
"""Delete back-end database"""
if self.db_name in self.server.listDatabases():
self.server.deleteDatabase(self.db_name)
def check(self, request, mtype=None):
"""Check that given request name is valid"""
# TODO: we may add some logic to check request name, etc.
if not request:
raise LogDBError("Request name is empty")
if mtype and mtype not in LOGDB_MSG_TYPES:
raise LogDBError("Unsupported message type: '%s', supported types %s" \
% (mtype, LOGDB_MSG_TYPES))
def docid(self, request, mtype):
"""Generate doc id, we use double dash to avoid dashes from thread names"""
return gen_hash('--'.join((request, self.dbid, self.thread_name, mtype)))
def prefix(self, mtype):
"""Generate agent specific prefix for given message type"""
if self.agent:
# we add prefix for agent messages, all others will not have this index
mtype = 'agent-%s' % mtype
return mtype
def agent_update(self, request, msg='', mtype="info"):
"""Update agent info in LogDB for given request"""
self.check(request, mtype)
mtype = self.prefix(mtype)
rec = {"ts":tstamp(), "msg":msg}
doc = {"_id": self.docid(request, mtype), "messages": [rec],
"request":request, "identifier":self.dbid,
"thr":self.thread_name, "type":mtype}
try:
exist_doc = self.db.document(doc["_id"])
doc["_rev"] = exist_doc["_rev"]
except CouchNotFoundError:
# this means document is not exist so we will just insert
pass
finally:
res = self.db.commitOne(doc)
return res
def user_update(self, request, msg, mtype='comment'):
"""Update user info in LogDB for given request"""
rec = {"ts":tstamp(), "msg":msg}
doc = {"_id": self.docid(request, mtype), "messages": [rec],
"request":request, "identifier":self.dbid,
"thr":self.thread_name, "type":mtype}
try:
exist_doc = self.db.document(doc["_id"])
doc["_rev"] = exist_doc["_rev"]
doc["messages"] += exist_doc["messages"]
except CouchNotFoundError:
# this means document is not exist so we will just insert
pass
finally:
res = self.db.commitOne(doc)
return res
def get(self, request, mtype=None, detail=True, agent=True):
"""Retrieve all entries from LogDB for given request"""
self.check(request, mtype)
if agent and mtype:
mtype = self.prefix(mtype)
options = {'reduce':False}
if mtype:
keys = [[request, mtype]]
else:
keys=[]
options.update({'startkey': [request], 'endkey':[request, {}]})
if detail:
options.update({'include_docs': True})
docs = self.db.loadView(self.design, self.view, options, keys=keys)
return docs
def get_by_thread(self, request, mtype='error', detail=False, agent=True):
self.check(request, mtype)
if agent and mtype:
mtype = self.prefix(mtype)
keys = [[request, self.dbid, self.thread_name, mtype]]
options = {'reduce':False}
if detail:
options.update({'include_docs': True})
docs = self.db.loadView(self.design, self.threadview, options, keys)
return docs
def get_by_request(self, request):
keys = [request]
options = {'reduce':False}
docs = self.db.loadView(self.design, self.requestview, options, keys)
return docs
def get_all_requests(self):
"""Retrieve all entries from LogDB"""
options = {'reduce':True, 'group_level':1}
docs = self.db.loadView(self.design, self.view, options)
return docs
def delete(self, request, mtype=None, this_thread=False, agent=True):
"""Delete entry in LogDB for given request"""
if mtype:
self.check(request, mtype)
else:
self.check(request)
if this_thread:
docs = self.get_by_thread(request, mtype=mtype, detail=False, agent=agent)
else:
docs = self.get(request, mtype=mtype, detail=False, agent=agent)
ids = [r['id'] for r in docs.get('rows', [])]
res = self.db.bulkDeleteByIDs(ids)
return res
def cleanup(self, thr):
"""
Clean-up docs older then given threshold (thr should be specified in seconds).
This is done via tstamp view end endkey, e.g.
curl "http://127.0.0.1:5984/logdb/_design/LogDB/_view/tstamp?endkey=1427912282"
"""
cutoff = round(time.time()-thr)
#docs = self.db.allDocs() # may need another view to look-up old docs
spec = {'endkey':cutoff, 'reduce':False}
docs = self.db.loadView(self.design, self.tsview, spec)
ids = [d['id'] for d in docs.get('rows', [])]
self.db.bulkDeleteByIDs(ids)
class LogDBBackend(object):
"""
Represents persistent storage for LogDB
"""
def __init__(self, db_url, db_name, identifier, thread_name, **kwds):
self.db_url = db_url
self.server = CouchServer(db_url)
self.db_name = db_name
self.dbid = identifier
self.thread_name = thread_name
self.agent = kwds.get('agent', 0)
self.db = self.server.connectDatabase(db_name, create=False)
self.design = 'LogDB' # name of design document
self.view = 'requests' # name of view to look-up requests
self.tsview = 'tstamp' # name of tsview to look-up requests
self.threadview = 'logByRequestAndThread'
self.requestview = 'logByRequest'
def deleteDatabase(self):
"""Delete back-end database"""
if self.db_name in self.server.listDatabases():
self.server.deleteDatabase(self.db_name)
def check(self, request, mtype=None):
"""Check that given request name is valid"""
# TODO: we may add some logic to check request name, etc.
if not request:
raise LogDBError("Request name is empty")
if mtype and mtype not in LOGDB_MSG_TYPES:
raise LogDBError("Unsupported message type: '%s', supported types %s" \
% (mtype, LOGDB_MSG_TYPES))
def docid(self, request, mtype):
"""Generate doc id, we use double dash to avoid dashes from thread names"""
return gen_hash('--'.join(
(request, self.dbid, self.thread_name, mtype)))
def prefix(self, mtype):
"""Generate agent specific prefix for given message type"""
if self.agent:
# we add prefix for agent messages, all others will not have this index
mtype = 'agent-%s' % mtype
return mtype
def agent_update(self, request, msg='', mtype="info"):
"""Update agent info in LogDB for given request"""
self.check(request, mtype)
mtype = self.prefix(mtype)
rec = {"ts": tstamp(), "msg": msg}
doc = {
"_id": self.docid(request, mtype),
"messages": [rec],
"request": request,
"identifier": self.dbid,
"thr": self.thread_name,
"type": mtype
}
try:
exist_doc = self.db.document(doc["_id"])
doc["_rev"] = exist_doc["_rev"]
except CouchNotFoundError:
# this means document is not exist so we will just insert
pass
finally:
res = self.db.commitOne(doc)
return res
def user_update(self, request, msg, mtype='comment'):
"""Update user info in LogDB for given request"""
rec = {"ts": tstamp(), "msg": msg}
doc = {
"_id": self.docid(request, mtype),
"messages": [rec],
"request": request,
"identifier": self.dbid,
"thr": self.thread_name,
"type": mtype
}
try:
exist_doc = self.db.document(doc["_id"])
doc["_rev"] = exist_doc["_rev"]
doc["messages"] += exist_doc["messages"]
except CouchNotFoundError:
# this means document is not exist so we will just insert
pass
finally:
res = self.db.commitOne(doc)
return res
def get(self, request, mtype=None, detail=True, agent=True):
"""Retrieve all entries from LogDB for given request"""
self.check(request, mtype)
if agent and mtype:
mtype = self.prefix(mtype)
options = {'reduce': False}
if mtype:
keys = [[request, mtype]]
else:
keys = []
options.update({'startkey': [request], 'endkey': [request, {}]})
if detail:
options.update({'include_docs': True})
docs = self.db.loadView(self.design, self.view, options, keys=keys)
return docs
def get_by_thread(self, request, mtype='error', detail=False, agent=True):
self.check(request, mtype)
if agent and mtype:
mtype = self.prefix(mtype)
keys = [[request, self.dbid, self.thread_name, mtype]]
options = {'reduce': False}
if detail:
options.update({'include_docs': True})
docs = self.db.loadView(self.design, self.threadview, options, keys)
return docs
def get_by_request(self, request):
keys = [request]
options = {'reduce': False}
docs = self.db.loadView(self.design, self.requestview, options, keys)
return docs
def get_all_requests(self):
"""Retrieve all entries from LogDB"""
options = {'reduce': True, 'group_level': 1}
docs = self.db.loadView(self.design, self.view, options)
return docs
def delete(self, request, mtype=None, this_thread=False, agent=True):
"""Delete entry in LogDB for given request"""
if mtype:
self.check(request, mtype)
else:
self.check(request)
if this_thread:
docs = self.get_by_thread(request,
mtype=mtype,
detail=False,
agent=agent)
else:
docs = self.get(request, mtype=mtype, detail=False, agent=agent)
ids = [r['id'] for r in docs.get('rows', [])]
res = self.db.bulkDeleteByIDs(ids)
return res
def cleanup(self, thr):
"""
Clean-up docs older then given threshold (thr should be specified in seconds).
This is done via tstamp view end endkey, e.g.
curl "http://127.0.0.1:5984/logdb/_design/LogDB/_view/tstamp?endkey=1427912282"
"""
cutoff = round(time.time() - thr)
#docs = self.db.allDocs() # may need another view to look-up old docs
spec = {'endkey': cutoff, 'reduce': False}
docs = self.db.loadView(self.design, self.tsview, spec)
ids = [d['id'] for d in docs.get('rows', [])]
self.db.bulkDeleteByIDs(ids)
class CMSCouchTest(unittest.TestCase):
test_counter = 0
def setUp(self):
# Make an instance of the server
self.server = CouchServer(
os.getenv("COUCHURL", 'http://*****:*****@localhost:5984'))
self.testname = self.id().split('.')[-1]
# Create a database, drop an existing one first
dbname = 'cmscouch_unittest_%s' % self.testname.lower()
if dbname in self.server.listDatabases():
self.server.deleteDatabase(dbname)
self.server.createDatabase(dbname)
self.db = self.server.connectDatabase(dbname)
def tearDown(self):
if sys.exc_info()[0] == None:
# This test has passed, clean up after it
dbname = 'cmscouch_unittest_%s' % self.testname.lower()
self.server.deleteDatabase(dbname)
def testCommitOne(self):
# Can I commit one dict
doc = {'foo': 123, 'bar': 456}
id = self.db.commitOne(doc, returndocs=True)[0]['id']
# What about a Document
doc = Document(inputDict=doc)
id = self.db.commitOne(doc, returndocs=True)[0]['id']
def testCommitOneWithQueue(self):
"""
CommitOne bypasses the queue, but it should maintain the queue if
present for a future call to commit.
"""
# Queue up five docs
doc = {'foo': 123, 'bar': 456}
for i in range(1, 6):
self.db.queue(doc)
# Commit one Document
doc = Document(inputDict=doc)
id = self.db.commitOne(doc, returndocs=True)[0]['id']
self.assertEqual(1, len(self.db.allDocs()['rows']))
self.db.commit()
self.assertEqual(6, len(self.db.allDocs()['rows']))
def testTimeStamping(self):
doc = {'foo': 123, 'bar': 456}
id = self.db.commitOne(doc, timestamp=True, returndocs=True)[0]['id']
doc = self.db.document(id)
self.assertTrue('timestamp' in doc.keys())
def testDeleteDoc(self):
doc = {'foo': 123, 'bar': 456}
self.db.commitOne(doc)
all_docs = self.db.allDocs()
self.assertEqual(1, len(all_docs['rows']))
# The db.delete_doc is immediate
id = all_docs['rows'][0]['id']
self.db.delete_doc(id)
all_docs = self.db.allDocs()
self.assertEqual(0, len(all_docs['rows']))
def testDeleteQueuedDocs(self):
doc1 = {'foo': 123, 'bar': 456}
doc2 = {'foo': 789, 'bar': 101112}
self.db.queue(doc1)
self.db.queue(doc2)
self.db.commit()
all_docs = self.db.allDocs()
self.assertEqual(2, len(all_docs['rows']))
for res in all_docs['rows']:
id = res['id']
doc = self.db.document(id)
self.db.queueDelete(doc)
all_docs = self.db.allDocs()
self.assertEqual(2, len(all_docs['rows']))
self.db.commit()
all_docs = self.db.allDocs()
self.assertEqual(0, len(all_docs['rows']))
def testWriteReadDocNoID(self):
doc = {}
def testReplicate(self):
repl_db = self.server.connectDatabase(self.db.name + 'repl')
doc_id = self.db.commitOne({'foo': 123},
timestamp=True,
returndocs=True)[0]['id']
doc_v1 = self.db.document(doc_id)
#replicate
self.server.replicate(self.db.name, repl_db.name)
self.assertEqual(self.db.document(doc_id), repl_db.document(doc_id))
self.server.deleteDatabase(repl_db.name)
def testSlashInDBName(self):
"""
Slashes are a valid character in a database name, and are useful as it
creates a directory strucutre for the couch data files.
"""
db_name = 'wmcore/unittests'
try:
self.server.deleteDatabase(db_name)
except:
# Ignore this - the database shouldn't already exist
pass
db = self.server.createDatabase(db_name)
info = db.info()
assert info['db_name'] == db_name
db_name = 'wmcore/unittests'
db = self.server.connectDatabase(db_name)
info = db.info()
assert info['db_name'] == db_name
db = Database(db_name, url=os.environ["COUCHURL"])
info = db.info()
assert info['db_name'] == db_name
self.server.deleteDatabase(db_name)
def testInvalidName(self):
"""
Capitol letters are not allowed in database names.
"""
db_name = 'Not A Valid Name'
self.assertRaises(ValueError, self.server.createDatabase, db_name)
self.assertRaises(ValueError, self.server.deleteDatabase, db_name)
self.assertRaises(ValueError, self.server.connectDatabase, db_name)
self.assertRaises(ValueError, Database, db_name)
def testDocumentSerialisation(self):
"""
A document should be writable into the couchdb with a timestamp.
"""
d = Document()
d['foo'] = 'bar'
doc_info = self.db.commit(doc=d, timestamp=True)[0]
d_from_db = self.db.document(doc_info['id'])
self.assertEqual(d['foo'], d_from_db['foo'])
self.assertEqual(d['timestamp'], d_from_db['timestamp'])
def testAttachments(self):
"""
Test uploading attachments with and without checksumming
"""
doc = self.db.commitOne({'foo': 'bar'},
timestamp=True,
returndocs=True)[0]
attachment1 = "Hello"
attachment2 = "How are you today?"
attachment3 = "I'm very well, thanks for asking"
attachment4 = "Lovely weather we're having"
attachment5 = "Goodbye"
keyhash = hashlib.md5()
keyhash.update(attachment5)
attachment5_md5 = keyhash.digest()
attachment5_md5 = base64.b64encode(attachment5_md5)
attachment6 = "Good day to you, sir!"
#TODO: add a binary attachment - e.g. tar.gz
doc = self.db.addAttachment(doc['id'], doc['rev'], attachment1)
doc = self.db.addAttachment(doc['id'],
doc['rev'],
attachment2,
contentType="foo/bar")
doc = self.db.addAttachment(doc['id'],
doc['rev'],
attachment3,
name="my_greeting")
doc = self.db.addAttachment(doc['id'],
doc['rev'],
attachment4,
add_checksum=True)
doc = self.db.addAttachment(doc['id'],
doc['rev'],
attachment5,
checksum=attachment5_md5)
self.assertRaises(CouchInternalServerError,
self.db.addAttachment,
doc['id'],
doc['rev'],
attachment6,
checksum='123')
def testRevisionHandling(self):
# This test won't work from an existing database, conflicts will be preserved, so
# ruthlessly remove the databases to get a clean slate.
try:
self.server.deleteDatabase(self.db.name)
except CouchNotFoundError:
pass # Must have been deleted already
try:
self.server.deleteDatabase(self.db.name + 'repl')
except CouchNotFoundError:
pass # Must have been deleted already
# I'm going to create a conflict, so need a replica db
self.db = self.server.connectDatabase(self.db.name)
repl_db = self.server.connectDatabase(self.db.name + 'repl')
doc_id = self.db.commitOne({'foo': 123},
timestamp=True,
returndocs=True)[0]['id']
doc_v1 = self.db.document(doc_id)
#replicate
self.server.replicate(self.db.name, repl_db.name)
doc_v2 = self.db.document(doc_id)
doc_v2['bar'] = 456
doc_id_rev2 = self.db.commitOne(doc_v2, returndocs=True)[0]
doc_v2 = self.db.document(doc_id)
#now update the replica
conflict_doc = repl_db.document(doc_id)
conflict_doc['bar'] = 101112
repl_db.commitOne(conflict_doc)
#replicate, creating the conflict
self.server.replicate(self.db.name, repl_db.name)
conflict_view = {
'map':
"function(doc) {if(doc._conflicts) {emit(doc._conflicts, null);}}"
}
data = repl_db.post('/%s/_temp_view' % repl_db.name, conflict_view)
# Should have one conflict in the repl database
self.assertEqual(data['total_rows'], 1)
# Should have no conflicts in the source database
self.assertEqual(
self.db.post('/%s/_temp_view' % self.db.name,
conflict_view)['total_rows'], 0)
self.assertTrue(
repl_db.documentExists(data['rows'][0]['id'],
rev=data['rows'][0]['key'][0]))
repl_db.delete_doc(data['rows'][0]['id'],
rev=data['rows'][0]['key'][0])
data = repl_db.post('/%s/_temp_view' % repl_db.name, conflict_view)
self.assertEqual(data['total_rows'], 0)
self.server.deleteDatabase(repl_db.name)
#update it again
doc_v3 = self.db.document(doc_id)
doc_v3['baz'] = 789
doc_id_rev3 = self.db.commitOne(doc_v3, returndocs=True)[0]
doc_v3 = self.db.document(doc_id)
#test that I can pull out an old revision
doc_v1_test = self.db.document(doc_id, rev=doc_v1['_rev'])
self.assertEqual(doc_v1, doc_v1_test)
#test that I can check a revision exists
self.assertTrue(self.db.documentExists(doc_id, rev=doc_v2['_rev']))
self.assertFalse(
self.db.documentExists(doc_id, rev='1' + doc_v2['_rev']))
#why you shouldn't rely on rev
self.db.compact(blocking=True)
self.assertFalse(self.db.documentExists(doc_id, rev=doc_v1['_rev']))
self.assertFalse(self.db.documentExists(doc_id, rev=doc_v2['_rev']))
self.assertTrue(self.db.documentExists(doc_id, rev=doc_v3['_rev']))
def testCommit(self):
"""
Test queue and commit modes
"""
# try to commit 2 random docs
doc = {'foo': 123, 'bar': 456}
self.db.queue(doc)
self.db.queue(doc)
self.assertEqual(2, len(self.db.commit()))
# committing 2 docs with the same id will fail
self.db.queue(Document(id="1", inputDict={'foo': 123, 'bar': 456}))
self.db.queue(Document(id="1", inputDict={'foo': 1234, 'bar': 456}))
answer = self.db.commit()
self.assertEqual(2, len(answer))
self.assertEqual(answer[0]['error'], 'conflict')
self.assertEqual(answer[1]['error'], 'conflict')
# all_or_nothing mode ignores conflicts
self.db.queue(Document(id="2", inputDict=doc))
self.db.queue(Document(id="2", inputDict={'foo': 1234, 'bar': 456}))
answer = self.db.commit(all_or_nothing=True)
self.assertEqual(2, len(answer))
self.assertEqual(answer[0].get('error'), None)
self.assertEqual(answer[0].get('error'), None)
self.assertEqual(answer[0]['id'], '2')
self.assertEqual(answer[1]['id'], '2')
# callbacks can do stuff when conflicts arise
# this particular one just overwrites the document
def callback(db, data, result):
for doc in data['docs']:
if doc['_id'] == result['id']:
doc['_rev'] = db.document(doc['_id'])['_rev']
retval = db.commitOne(doc)
return retval[0]
self.db.queue(Document(id="2", inputDict={'foo': 5, 'bar': 6}))
answer = self.db.commit(callback=callback)
self.assertEqual(1, len(answer))
self.assertEqual(answer[0].get('error'), None)
updatedDoc = self.db.document('2')
self.assertEqual(updatedDoc['foo'], 5)
self.assertEqual(updatedDoc['bar'], 6)
return
def testUpdateHandler(self):
"""
Test that update function support works
"""
update_ddoc = {
'_id': '_design/foo',
'language': 'javascript',
'updates': {
"bump-counter":
'function(doc, req) {if (!doc.counter) {doc.counter = 0};doc.counter += 1;return [doc,"bumped it!"];}',
}
}
self.db.commit(update_ddoc)
doc = {'foo': 123, 'counter': 0}
doc_id = self.db.commit(doc)[0]['id']
self.assertEqual("bumped it!",
self.db.updateDocument(doc_id, 'foo', 'bump-counter'))
self.assertEqual(1, self.db.document(doc_id)['counter'])
def testList(self):
"""
Test list function works ok
"""
update_ddoc = {
'_id': '_design/foo',
'language': 'javascript',
'views': {
'all': {
'map': 'function(doc) {emit(null, null) }'
},
},
'lists': {
'errorinoutput':
'function(doc, req) {send("A string with the word error in")}',
'malformed': 'function(doc, req) {somethingtoraiseanerror}',
}
}
self.db.commit(update_ddoc)
# approriate errors raised
self.assertRaises(CouchNotFoundError, self.db.loadList, 'foo', 'error',
'view_doesnt_exist')
self.assertRaises(CouchInternalServerError, self.db.loadList, 'foo',
'malformed', 'all')
# error in list output string shouldn't raise an error
self.assertEqual(self.db.loadList('foo', 'errorinoutput', 'all'),
"A string with the word error in")
def testAllDocs(self):
"""
Test AllDocs with options
"""
self.db.queue(Document(id="1", inputDict={'foo': 123, 'bar': 456}))
self.db.queue(Document(id="2", inputDict={'foo': 123, 'bar': 456}))
self.db.queue(Document(id="3", inputDict={'foo': 123, 'bar': 456}))
self.db.commit()
self.assertEqual(3, len(self.db.allDocs()['rows']))
self.assertEqual(2, len(self.db.allDocs({'startkey': "2"})['rows']))
self.assertEqual(2, len(self.db.allDocs(keys=["1", "3"])['rows']))
self.assertEqual(
1, len(self.db.allDocs({'limit': 1}, ["1", "3"])['rows']))
self.assertTrue('error' in self.db.allDocs(keys=["1", "4"])['rows'][1])
def configure(self):
server = CouchServer(self.config.JobStateMachine.couchurl)
dbname = 'JSM/JobHistory'
if dbname not in server.listDatabases():
server.createDatabase(dbname)
class ConfigCache(WMObject):
"""
_ConfigCache_
The class that handles the upload and download of configCache
artifacts from Couch
"""
def __init__(self, dbURL, couchDBName = None, id = None, rev = None, usePYCurl = False,
ckey = None, cert = None, capath = None, detail = True):
self.dbname = couchDBName
self.dburl = dbURL
self.detail = detail
try:
self.couchdb = CouchServer(self.dburl, usePYCurl=usePYCurl, ckey=ckey, cert=cert, capath=capath)
if self.dbname not in self.couchdb.listDatabases():
self.createDatabase()
self.database = self.couchdb.connectDatabase(self.dbname)
except Exception as ex:
msg = "Error connecting to couch: %s\n" % str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise ConfigCacheException(message = msg)
# local cache
self.docs_cache = DocumentCache(self.database, self.detail)
# UserGroup variables
self.group = None
self.owner = None
# Internal data structure
self.document = Document()
self.attachments = {}
self.document['type'] = "config"
self.document['description'] = {}
self.document['description']['config_label'] = None
self.document['description']['config_desc'] = None
if id != None:
self.document['_id'] = id
self.document['pset_tweak_details'] = None
self.document['info'] = None
self.document['config'] = None
return
def createDatabase(self):
"""
_createDatabase_
"""
database = self.couchdb.createDatabase(self.dbname)
database.commit()
return database
def connectUserGroup(self, groupname, username):
"""
_connectUserGroup_
"""
self.group = Group(name = groupname)
self.group.setCouch(self.dburl, self.dbname)
self.group.connect()
self.owner = makeUser(groupname, username,
couchUrl = self.dburl,
couchDatabase = self.dbname)
return
def createUserGroup(self, groupname, username):
"""
_createUserGroup_
Create all the userGroup information
"""
self.createGroup(name = groupname)
self.createUser(username = username)
return
def createGroup(self, name):
"""
_createGroup_
Create Group for GroupUser
"""
self.group = Group(name = name)
self.group.setCouch(self.dburl, self.dbname)
self.group.connect()
self.group.create()
return
def setLabel(self, label):
"""
_setLabel_
Util to add a descriptive label to the configuration doc
"""
self.document['description']['config_label'] = label
def setDescription(self, desc):
"""
_setDescription_
Util to add a verbose description string to a configuration doc
"""
self.document['description']['config_desc'] = desc
@Decorators.requireGroup
def createUser(self, username):
self.owner = makeUser(self.group['name'], username,
couchUrl = self.dburl,
couchDatabase = self.dbname)
self.owner.create()
self.owner.ownThis(self.document)
return
@Decorators.requireGroup
@Decorators.requireUser
def save(self):
"""
_save_
Save yourself! Save your internal document.
"""
rawResults = self.database.commit(doc = self.document)
# We should only be committing one document at a time
# if not, get the last one.
try:
commitResults = rawResults[-1]
self.document["_rev"] = commitResults.get('rev')
self.document["_id"] = commitResults.get('id')
except KeyError as ex:
msg = "Document returned from couch without ID or Revision\n"
msg += "Document probably bad\n"
msg += str(ex)
logging.error(msg)
raise ConfigCacheException(message = msg)
# Now do the attachments
for attachName in self.attachments:
self.saveAttachment(name = attachName,
attachment = self.attachments[attachName])
return
def saveAttachment(self, name, attachment):
"""
_saveAttachment_
Save an attachment to the document
"""
retval = self.database.addAttachment(self.document["_id"],
self.document["_rev"],
attachment,
name)
if retval.get('ok', False) != True:
# Then we have a problem
msg = "Adding an attachment to document failed\n"
msg += str(retval)
msg += "ID: %s, Rev: %s" % (self.document["_id"], self.document["_rev"])
logging.error(msg)
raise ConfigCacheException(msg)
self.document["_rev"] = retval['rev']
self.document["_id"] = retval['id']
self.attachments[name] = attachment
return
def loadDocument(self, configID):
"""
_loadDocument_
Load a document from the document cache given its couchID
"""
self.document = self.docs_cache[configID]
def loadByID(self, configID):
"""
_loadByID_
Load a document from the server given its couchID
"""
try:
self.document = self.database.document(id = configID)
if 'owner' in self.document.keys():
self.connectUserGroup(groupname = self.document['owner'].get('group', None),
username = self.document['owner'].get('user', None))
if '_attachments' in self.document.keys():
# Then we need to load the attachments
for key in self.document['_attachments'].keys():
self.loadAttachment(name = key)
except CouchNotFoundError as ex:
msg = "Document with id %s not found in couch\n" % (configID)
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise ConfigCacheException(message = msg)
except Exception as ex:
msg = "Error loading document from couch\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise ConfigCacheException(message = msg)
return
def loadAttachment(self, name, overwrite = True):
"""
_loadAttachment_
Load an attachment from the database and put it somewhere useful
"""
attach = self.database.getAttachment(self.document["_id"], name)
if not overwrite:
if name in self.attachments.keys():
logging.info("Attachment already exists, so we're skipping")
return
self.attachments[name] = attach
return
def loadByView(self, view, value):
"""
_loadByView_
Underlying code to load views
"""
viewRes = self.database.loadView( 'ConfigCache', view, {}, [value] )
if len(viewRes['rows']) == 0:
# Then we have a problem
logging.error("Unable to load using view %s and value %s" % (view, str(value)))
self.unwrapView(viewRes)
self.loadByID(self.document["_id"])
return
def saveConfigToDisk(self, targetFile):
"""
_saveConfigToDisk_
Make sure we can save our config file to disk
"""
config = self.getConfig()
if not config:
return
# Write to a file
f = open(targetFile, 'w')
f.write(config)
f.close()
return
def load(self):
"""
_load_
Figure out how to load
"""
if self.document.get("_id", None) != None:
# Then we should load by ID
self.loadByID(self.document["_id"])
return
# Otherwise we have to load by view
if not self.document.get('md5_hash', None) == None:
# Then we have an md5_hash
self.loadByView(view = 'config_by_md5hash', value = self.document['md5_hash'])
# TODO: Add more views as they become available.
#elif not self.owner == None:
# Then we have an owner
#self.loadByView(view = 'config_by_owner', value = self.owner['name'])
def unwrapView(self, view):
"""
_unwrapView_
Move view information into the main document
"""
self.document["_id"] = view['rows'][0].get('id')
self.document["_rev"] = view['rows'][0].get('value').get('_rev')
def setPSetTweaks(self, PSetTweak):
"""
_setPSetTweaks_
Set the PSet tweak details for the config.
"""
self.document['pset_tweak_details'] = PSetTweak
return
def getPSetTweaks(self):
"""
_getPSetTweaks_
Retrieve the PSet tweak details.
"""
return self.document['pset_tweak_details']
def getOutputModuleInfo(self):
"""
_getOutputModuleInfo_
Retrieve the dataset information for the config in the ConfigCache.
"""
psetTweaks = self.getPSetTweaks()
if not 'process' in psetTweaks.keys():
raise ConfigCacheException("Could not find process field in PSet while getting output modules!")
try:
outputModuleNames = psetTweaks["process"]["outputModules_"]
except KeyError as ex:
msg = "Could not find outputModules_ in psetTweaks['process'] while getting output modules.\n"
msg += str(ex)
logging.error(msg)
raise ConfigCacheException(msg)
results = {}
for outputModuleName in outputModuleNames:
try:
outModule = psetTweaks["process"][outputModuleName]
except KeyError:
msg = "Could not find outputModule %s in psetTweaks['process']" % outputModuleName
logging.error(msg)
raise ConfigCacheException(msg)
dataset = outModule.get("dataset", None)
if dataset:
results[outputModuleName] = {"dataTier": outModule["dataset"]["dataTier"],
"filterName": outModule["dataset"]["filterName"]}
else:
results[outputModuleName] = {"dataTier": None, "filterName": None}
return results
def addConfig(self, newConfig, psetHash = None):
"""
_addConfig_
"""
# The newConfig parameter is a URL suitable for passing to urlopen.
configString = urllib.urlopen(newConfig).read(-1)
configMD5 = hashlib.md5(configString).hexdigest()
self.document['md5_hash'] = configMD5
self.document['pset_hash'] = psetHash
self.attachments['configFile'] = configString
return
def getConfig(self):
"""
_getConfig_
Get the currently active config
"""
return self.attachments.get('configFile', None)
def getCouchID(self):
"""
_getCouchID_
Return the document's couchID
"""
return self.document["_id"]
def getCouchRev(self):
"""
_getCouchRev_
Return the document's couchRevision
"""
return self.document["_rev"]
@Decorators.requireGroup
@Decorators.requireUser
def delete(self):
"""
_delete_
Deletes the document with the current docid
"""
if not self.document["_id"]:
logging.error("Attempted to delete with no couch ID")
# TODO: Delete without loading first
try:
self.database.queueDelete(self.document)
self.database.commit()
except Exception as ex:
msg = "Error in deleting document from couch"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise ConfigCacheException(message = msg)
return
def getIDFromLabel(self, label):
"""
_getIDFromLabel_
Retrieve the ID of a config given it's label.
"""
results = self.database.loadView("ConfigCache", "config_by_label",
{"startkey": label,
"limit": 1})
if results["rows"][0]["key"] == label:
return results["rows"][0]["value"]
return None
def listAllConfigsByLabel(self):
"""
_listAllConfigsByLabel_
Retrieve a list of all the configs in the config cache. This is
returned in the form of a dictionary that is keyed by label.
"""
configs = {}
results = self.database.loadView("ConfigCache", "config_by_label")
for result in results["rows"]:
configs[result["key"]] = result["value"]
return configs
def __str__(self):
"""
Make something printable
"""
return self.document.__str__()
def validate(self, configID):
try:
#TODO: need to change to DataCache
#self.loadDocument(configID = configID)
self.loadByID(configID = configID)
except Exception as ex:
raise ConfigCacheException("Failure to load ConfigCache while validating workload: %s" % str(ex))
if self.detail:
duplicateCheck = {}
try:
outputModuleInfo = self.getOutputModuleInfo()
except Exception as ex:
# Something's gone wrong with trying to open the configCache
msg = "Error in getting output modules from ConfigCache during workload validation. Check ConfigCache formatting!"
raise ConfigCacheException("%s: %s" % (msg, str(ex)))
for outputModule in outputModuleInfo.values():
dataTier = outputModule.get('dataTier', None)
filterName = outputModule.get('filterName', None)
if not dataTier:
raise ConfigCacheException("No DataTier in output module.")
# Add dataTier to duplicate dictionary
if not dataTier in duplicateCheck.keys():
duplicateCheck[dataTier] = []
if filterName in duplicateCheck[dataTier]:
# Then we've seen this combination before
raise ConfigCacheException("Duplicate dataTier/filterName combination.")
else:
duplicateCheck[dataTier].append(filterName)
return outputModuleInfo
else:
return True
class LogDBBackend(object):
"""
Represents persistent storage for LogDB
"""
def __init__(self, db_url, db_name, identifier, thread_name, **kwds):
self.db_url = db_url
self.server = CouchServer(db_url)
self.db_name = db_name
self.dbid = identifier
self.thread_name = thread_name
self.agent = kwds.get('agent', 0)
create = kwds.get('create', False)
size = kwds.get('size', 10000)
self.db = self.server.connectDatabase(db_name, create=create, size=size)
self.design = kwds.get('design', 'LogDB') # name of design document
self.view = kwds.get('view', 'requests') # name of view to look-up requests
self.tsview = kwds.get('tsview', 'tstamp') # name of tsview to look-up requests
if create:
uri = '/%s/_design/%s' % (db_name, self.design)
data = design_doc()
try:
# insert design doc, if fails due to conflict continue
# conflict may happen due to concurrent client connection who
# created first this doc
self.db.put(uri, data)
except CouchConflictError:
pass
def deleteDatabase(self):
"""Delete back-end database"""
if self.db_name in self.server.listDatabases():
self.server.deleteDatabase(self.db_name)
def check(self, request, mtype=None):
"""Check that given request name is valid"""
# TODO: we may add some logic to check request name, etc.
if not request:
raise LogDBError("Request name is empty")
if mtype and mtype not in LOGDB_MSG_TYPES:
raise LogDBError("Unsupported message type: '%s', supported types %s" \
% (mtype, LOGDB_MSG_TYPES))
def docid(self, request, mtype):
"""Generate doc id, we use double dash to avoid dashes from thread names"""
return gen_hash('--'.join((request, self.dbid, self.thread_name, mtype)))
def prefix(self, mtype):
"""Generate agent specific prefix for given message type"""
if self.agent:
# we add prefix for agent messages, all others will not have this index
mtype = 'agent-%s' % mtype
return mtype
def agent_update(self, request, msg='', mtype="info"):
"""Update agent info in LogDB for given request"""
self.check(request, mtype)
mtype = self.prefix(mtype)
rec = {"ts":tstamp(), "msg":msg}
doc = {"_id": self.docid(request, mtype), "messages": [rec],
"request":request, "identifier":self.dbid,
"thr":self.thread_name, "type":mtype}
try:
exist_doc = self.db.document(doc["_id"])
doc["_rev"] = exist_doc["_rev"]
except CouchNotFoundError:
# this means document is not exist so we will just insert
pass
finally:
res = self.db.commitOne(doc)
return res
def user_update(self, request, msg, mtype='comment'):
"""Update user info in LogDB for given request"""
rec = {"ts":tstamp(), "msg":msg}
doc = {"_id": self.docid(request, mtype), "messages": [rec],
"request":request, "identifier":self.dbid,
"thr":self.thread_name, "type":mtype}
try:
exist_doc = self.db.document(doc["_id"])
doc["_rev"] = exist_doc["_rev"]
doc["messages"] += exist_doc["messages"]
except CouchNotFoundError:
# this means document is not exist so we will just insert
pass
finally:
res = self.db.commitOne(doc)
return res
def get(self, request, mtype=None, detail=True):
"""Retrieve all entries from LogDB for given request"""
self.check(request, mtype)
spec = {'request':request, 'reduce':False}
if mtype:
spec.update({'type':mtype})
if detail:
spec.update({'include_docs': True})
docs = self.db.loadView(self.design, self.view, spec)
return docs
def get_all_requests(self):
"""Retrieve all entries from LogDB"""
spec = {'reduce':True, 'group_level':1}
docs = self.db.loadView(self.design, self.view, spec)
return docs
def delete(self, request):
"""Delete entry in LogDB for given request"""
self.check(request)
docs = self.get(request, detail=False)
ids = [r['id'] for r in docs.get('rows', [])]
res = self.db.bulkDeleteByIDs(ids)
return res
def cleanup(self, thr):
"""
Clean-up docs older then given threshold (thr should be specified in seconds).
This is done via tstamp view end endkey, e.g.
curl "http://127.0.0.1:5984/logdb/_design/LogDB/_view/tstamp?endkey=1427912282"
"""
tstamp = round(time.time()-thr)
docs = self.db.allDocs() # may need another view to look-up old docs
spec = {'endkey':tstamp, 'reduce':False}
docs = self.db.loadView(self.design, self.tsview, spec)
ids = [d['id'] for d in docs.get('rows', [])]
self.db.bulkDeleteByIDs(ids)
class CMSCouchTest(unittest.TestCase):
test_counter = 0
def setUp(self):
# Make an instance of the server
self.server = CouchServer(os.getenv("COUCHURL", 'http://*****:*****@localhost:5984'))
self.testname = self.id().split('.')[-1]
# Create a database, drop an existing one first
dbname = 'cmscouch_unittest_%s' % self.testname.lower()
if dbname in self.server.listDatabases():
self.server.deleteDatabase(dbname)
self.server.createDatabase(dbname)
self.db = self.server.connectDatabase(dbname)
def tearDown(self):
if sys.exc_info()[0] == None:
# This test has passed, clean up after it
dbname = 'cmscouch_unittest_%s' % self.testname.lower()
self.server.deleteDatabase(dbname)
def testCommitOne(self):
# Can I commit one dict
doc = {'foo':123, 'bar':456}
id = self.db.commitOne(doc, returndocs=True)[0]['id']
# What about a Document
doc = Document(inputDict = doc)
id = self.db.commitOne(doc, returndocs=True)[0]['id']
def testCommitOneWithQueue(self):
"""
CommitOne bypasses the queue, but it should maintain the queue if
present for a future call to commit.
"""
# Queue up five docs
doc = {'foo':123, 'bar':456}
for i in range(1,6):
self.db.queue(doc)
# Commit one Document
doc = Document(inputDict = doc)
id = self.db.commitOne(doc, returndocs=True)[0]['id']
self.assertEqual(1, len(self.db.allDocs()['rows']))
self.db.commit()
self.assertEqual(6, len(self.db.allDocs()['rows']))
def testTimeStamping(self):
doc = {'foo':123, 'bar':456}
id = self.db.commitOne(doc, timestamp=True, returndocs=True)[0]['id']
doc = self.db.document(id)
self.assertTrue('timestamp' in doc.keys())
def testDeleteDoc(self):
doc = {'foo':123, 'bar':456}
self.db.commitOne(doc)
all_docs = self.db.allDocs()
self.assertEqual(1, len(all_docs['rows']))
# The db.delete_doc is immediate
id = all_docs['rows'][0]['id']
self.db.delete_doc(id)
all_docs = self.db.allDocs()
self.assertEqual(0, len(all_docs['rows']))
def testDeleteQueuedDocs(self):
doc1 = {'foo':123, 'bar':456}
doc2 = {'foo':789, 'bar':101112}
self.db.queue(doc1)
self.db.queue(doc2)
self.db.commit()
all_docs = self.db.allDocs()
self.assertEqual(2, len(all_docs['rows']))
for res in all_docs['rows']:
id = res['id']
doc = self.db.document(id)
self.db.queueDelete(doc)
all_docs = self.db.allDocs()
self.assertEqual(2, len(all_docs['rows']))
self.db.commit()
all_docs = self.db.allDocs()
self.assertEqual(0, len(all_docs['rows']))
def testWriteReadDocNoID(self):
doc = {}
def testReplicate(self):
repl_db = self.server.connectDatabase(self.db.name + 'repl')
doc_id = self.db.commitOne({'foo':123}, timestamp=True, returndocs=True)[0]['id']
doc_v1 = self.db.document(doc_id)
#replicate
self.server.replicate(self.db.name, repl_db.name)
self.assertEqual(self.db.document(doc_id), repl_db.document(doc_id))
self.server.deleteDatabase(repl_db.name)
def testSlashInDBName(self):
"""
Slashes are a valid character in a database name, and are useful as it
creates a directory strucutre for the couch data files.
"""
db_name = 'wmcore/unittests'
try:
self.server.deleteDatabase(db_name)
except:
# Ignore this - the database shouldn't already exist
pass
db = self.server.createDatabase(db_name)
info = db.info()
assert info['db_name'] == db_name
db_name = 'wmcore/unittests'
db = self.server.connectDatabase(db_name)
info = db.info()
assert info['db_name'] == db_name
db = Database(db_name)
info = db.info()
assert info['db_name'] == db_name
self.server.deleteDatabase(db_name)
def testInvalidName(self):
"""
Capitol letters are not allowed in database names.
"""
db_name = 'Not A Valid Name'
self.assertRaises(ValueError, self.server.createDatabase, db_name)
self.assertRaises(ValueError, self.server.deleteDatabase, db_name)
self.assertRaises(ValueError, self.server.connectDatabase, db_name)
self.assertRaises(ValueError, Database, db_name)
def testDocumentSerialisation(self):
"""
A document should be writable into the couchdb with a timestamp.
"""
d = Document()
d['foo'] = 'bar'
doc_info = self.db.commit(doc=d, timestamp=True)[0]
d_from_db = self.db.document(doc_info['id'])
self.assertEquals(d['foo'], d_from_db['foo'])
self.assertEquals(d['timestamp'], d_from_db['timestamp'])
def testAttachments(self):
"""
Test uploading attachments with and without checksumming
"""
doc = self.db.commitOne({'foo':'bar'}, timestamp=True, returndocs=True)[0]
attachment1 = "Hello"
attachment2 = "How are you today?"
attachment3 = "I'm very well, thanks for asking"
attachment4 = "Lovely weather we're having"
attachment5 = "Goodbye"
keyhash = hashlib.md5()
keyhash.update(attachment5)
attachment5_md5 = keyhash.digest()
attachment5_md5 = base64.b64encode(attachment5_md5)
attachment6 = "Good day to you, sir!"
#TODO: add a binary attachment - e.g. tar.gz
doc = self.db.addAttachment(doc['id'], doc['rev'], attachment1)
doc = self.db.addAttachment(doc['id'], doc['rev'], attachment2, contentType="foo/bar")
doc = self.db.addAttachment(doc['id'], doc['rev'], attachment3, name="my_greeting")
doc = self.db.addAttachment(doc['id'], doc['rev'], attachment4, add_checksum=True)
doc = self.db.addAttachment(doc['id'], doc['rev'], attachment5, checksum=attachment5_md5)
self.assertRaises(CouchInternalServerError, self.db.addAttachment, doc['id'], doc['rev'], attachment6, checksum='123')
def testRevisionHandling(self):
# This test won't work from an existing database, conflicts will be preserved, so
# ruthlessly remove the databases to get a clean slate.
try:
self.server.deleteDatabase(self.db.name)
except CouchNotFoundError:
pass # Must have been deleted already
try:
self.server.deleteDatabase(self.db.name + 'repl')
except CouchNotFoundError:
pass # Must have been deleted already
# I'm going to create a conflict, so need a replica db
self.db = self.server.connectDatabase(self.db.name)
repl_db = self.server.connectDatabase(self.db.name + 'repl')
doc_id = self.db.commitOne({'foo':123}, timestamp=True, returndocs=True)[0]['id']
doc_v1 = self.db.document(doc_id)
#replicate
self.server.replicate(self.db.name, repl_db.name)
doc_v2 = self.db.document(doc_id)
doc_v2['bar'] = 456
doc_id_rev2 = self.db.commitOne(doc_v2, returndocs=True)[0]
doc_v2 = self.db.document(doc_id)
#now update the replica
conflict_doc = repl_db.document(doc_id)
conflict_doc['bar'] = 101112
repl_db.commitOne(conflict_doc)
#replicate, creating the conflict
self.server.replicate(self.db.name, repl_db.name)
conflict_view = {'map':"function(doc) {if(doc._conflicts) {emit(doc._conflicts, null);}}"}
data = repl_db.post('/%s/_temp_view' % repl_db.name, conflict_view)
# Should have one conflict in the repl database
self.assertEquals(data['total_rows'], 1)
# Should have no conflicts in the source database
self.assertEquals(self.db.post('/%s/_temp_view' % self.db.name, conflict_view)['total_rows'], 0)
self.assertTrue(repl_db.documentExists(data['rows'][0]['id'], rev=data['rows'][0]['key'][0]))
repl_db.delete_doc(data['rows'][0]['id'], rev=data['rows'][0]['key'][0])
data = repl_db.post('/%s/_temp_view' % repl_db.name, conflict_view)
self.assertEquals(data['total_rows'], 0)
self.server.deleteDatabase(repl_db.name)
#update it again
doc_v3 = self.db.document(doc_id)
doc_v3['baz'] = 789
doc_id_rev3 = self.db.commitOne(doc_v3, returndocs=True)[0]
doc_v3 = self.db.document(doc_id)
#test that I can pull out an old revision
doc_v1_test = self.db.document(doc_id, rev=doc_v1['_rev'])
self.assertEquals(doc_v1, doc_v1_test)
#test that I can check a revision exists
self.assertTrue(self.db.documentExists(doc_id, rev=doc_v2['_rev']))
self.assertFalse(self.db.documentExists(doc_id, rev='1'+doc_v2['_rev']))
#why you shouldn't rely on rev
self.db.compact(blocking=True)
self.assertFalse(self.db.documentExists(doc_id, rev=doc_v1['_rev']))
self.assertFalse(self.db.documentExists(doc_id, rev=doc_v2['_rev']))
self.assertTrue(self.db.documentExists(doc_id, rev=doc_v3['_rev']))
def testCommit(self):
"""
Test queue and commit modes
"""
# try to commit 2 random docs
doc = {'foo':123, 'bar':456}
self.db.queue(doc)
self.db.queue(doc)
self.assertEqual(2, len(self.db.commit()))
# committing 2 docs with the same id will fail
self.db.queue(Document(id = "1", inputDict = {'foo':123, 'bar':456}))
self.db.queue(Document(id = "1", inputDict = {'foo':1234, 'bar':456}))
answer = self.db.commit()
self.assertEqual(2, len(answer))
self.assertEqual(answer[0]['error'], 'conflict')
self.assertEqual(answer[1]['error'], 'conflict')
# all_or_nothing mode ignores conflicts
self.db.queue(Document(id = "2", inputDict = doc))
self.db.queue(Document(id = "2", inputDict = {'foo':1234, 'bar':456}))
answer = self.db.commit(all_or_nothing = True)
self.assertEqual(2, len(answer))
self.assertEqual(answer[0].get('error'), None)
self.assertEqual(answer[0].get('error'), None)
self.assertEqual(answer[0]['id'], '2')
self.assertEqual(answer[1]['id'], '2')
def testUpdateHandler(self):
"""
Test that update function support works
"""
update_ddoc = {
'_id':'_design/foo',
'language': 'javascript',
'updates':{
"bump-counter" : 'function(doc, req) {if (!doc.counter) {doc.counter = 0};doc.counter += 1;return [doc,"bumped it!"];}',
}
}
self.db.commit(update_ddoc)
doc = {'foo': 123, 'counter': 0}
doc_id = self.db.commit(doc)[0]['id']
self.assertEquals("bumped it!", self.db.updateDocument(doc_id, 'foo', 'bump-counter'))
self.assertEquals(1, self.db.document(doc_id)['counter'])
class RotatingDatabaseTest(unittest.TestCase):
def setUp(self):
self.couchURL = os.getenv("COUCHURL")
self.server = CouchServer(self.couchURL)
# Kill off any databases left over from previous runs
for db in [db for db in self.server.listDatabases() if db.startswith('rotdb_unittest_')]:
try:
self.server.deleteDatabase(db)
except:
pass
# Create a database, drop an existing one first
testname = self.id().split('.')[-1].lower()
self.dbname = 'rotdb_unittest_%s' % testname
self.arcname = 'rotdb_unittest_%s_archive' % testname
self.seedname = 'rotdb_unittest_%s_seedcfg' % testname
# set a long value for times, tests do operations explicitly
self.timing = {'archive':timedelta(seconds=1), 'expire':timedelta(seconds=2)}
self.db = RotatingDatabase(dbname = self.dbname, url = self.couchURL,
archivename = self.arcname, timing = self.timing)
def tearDown(self):
testname = self.id().split('.')[-1].lower()
if sys.exc_info()[0] == None:
# This test has passed, clean up after it
to_go = [db for db in self.server.listDatabases() if db.startswith('rotdb_unittest_%s' % testname)]
for dbname in to_go:
try:
self.server.deleteDatabase(dbname)
except CouchNotFoundError:
# db has already gone
pass
def testRotate(self):
"""
Test that rotation works
"""
start_name = self.db.name
self.db._rotate()
end_name = self.db.name
databases = [db for db in self.server.listDatabases() if db.startswith('rotdb_unittest_')]
self.assertTrue(start_name in databases)
self.assertTrue(end_name in databases)
def testArchive(self):
"""
Test that archiving views works
"""
dummy_view = {'_id':'_design/foo', 'language': 'javascript','views':{
'bar':{'map':"function(doc) {if (doc.foo) {emit(doc.int, 1);}}", 'reduce':'_sum'}
}
}
archive_view = {'_id':'_design/foo', 'language': 'javascript','views':{
'bar':{'map':"function(doc) {emit(doc.key, doc.value);}", 'reduce':'_sum'}
}
}
seed_db = self.server.connectDatabase(self.seedname)
seed_db.commit(dummy_view)
# Need to have the timing long enough so the data isn't archived by accident
self.timing = {'archive':timedelta(seconds=1000), 'expire':timedelta(seconds=2000)}
self.db = RotatingDatabase(dbname = self.dbname, url = self.couchURL, views=['foo/bar'],
archivename = self.arcname, timing = self.timing)
self.db.archive_db.commitOne(archive_view)
runs = 5
docs = 5
for run in range(runs):
for i in range(docs):
self.db.queue({'foo':'bar', 'int': i, 'run': run})
self.db.commit()
self.db._rotate()
self.db._archive()
view_result = self.db.archive_db.loadView('foo','bar')
arch_sum = view_result['rows'][0]['value']
self.assertEqual(arch_sum, runs * docs)
def testExpire(self):
"""
Test that expiring databases works
"""
# rotate out the original db
self.db._rotate()
archived = self.db.archived_dbs()
self.assertEqual(1, len(archived), 'test not starting from clean state, bail!')
# Make sure the db has expired
sleep(2)
self.db._expire()
self.assertEqual(0, len(self.db.archived_dbs()))
self.assertFalse(archived[0] in self.server.listDatabases())
@attr("integration")
def testCycle(self):
"""
Test that committing data to different databases happens
This is a bit of a dodgy test - if timings go funny it will fail
"""
self.timing = {'archive':timedelta(seconds=0.5), 'expire':timedelta(seconds=1)}
self.db = RotatingDatabase(dbname = self.dbname, url = self.couchURL,
archivename = self.arcname, timing = self.timing)
my_name = self.db.name
self.db.commit({'foo':'bar'})
sleep(5)
self.db.commit({'foo':'bar'})
# the initial db should have expired by now
self.db.commit({'foo':'bar'})
self.assertFalse(my_name in self.server.listDatabases(), "")
class CMSCouchTest(unittest.TestCase):
test_counter = 0
def setUp(self):
# Make an instance of the server
self.server = CouchServer(os.getenv("COUCHURL", 'http://*****:*****@localhost:5984'))
self.testname = self.id().split('.')[-1]
# Create a database, drop an existing one first
dbname = 'cmscouch_unittest_%s' % self.testname.lower()
if dbname in self.server.listDatabases():
self.server.deleteDatabase(dbname)
self.server.createDatabase(dbname)
self.db = self.server.connectDatabase(dbname)
def tearDown(self):
if sys.exc_info()[0] == None:
# This test has passed, clean up after it
dbname = 'cmscouch_unittest_%s' % self.testname.lower()
self.server.deleteDatabase(dbname)
def testCommitOne(self):
# Can I commit one dict
doc = {'foo':123, 'bar':456}
id = self.db.commitOne(doc)[0]['id']
# What about a Document
doc = Document(inputDict = doc)
id = self.db.commitOne(doc)[0]['id']
def testCommitOneWithQueue(self):
"""
CommitOne bypasses the queue, but it should maintain the queue if
present for a future call to commit.
"""
# Queue up five docs
doc = {'foo':123, 'bar':456}
for i in range(1,6):
self.db.queue(doc)
# Commit one Document
doc = Document(inputDict = doc)
id = self.db.commitOne(doc)[0]['id']
self.assertEqual(1, len(self.db.allDocs()['rows']))
self.db.commit()
self.assertEqual(6, len(self.db.allDocs()['rows']))
def testTimeStamping(self):
doc = {'foo':123, 'bar':456}
id = self.db.commitOne(doc, timestamp=True)[0]['id']
doc = self.db.document(id)
self.assertTrue('timestamp' in doc.keys())
def testDeleteDoc(self):
doc = {'foo':123, 'bar':456}
self.db.commitOne(doc)
all_docs = self.db.allDocs()
self.assertEqual(1, len(all_docs['rows']))
# The db.delete_doc is immediate
id = all_docs['rows'][0]['id']
self.db.delete_doc(id)
all_docs = self.db.allDocs()
self.assertEqual(0, len(all_docs['rows']))
def testDeleteQueuedDocs(self):
doc1 = {'foo':123, 'bar':456}
doc2 = {'foo':789, 'bar':101112}
self.db.queue(doc1)
self.db.queue(doc2)
self.db.commit()
all_docs = self.db.allDocs()
self.assertEqual(2, len(all_docs['rows']))
for res in all_docs['rows']:
id = res['id']
doc = self.db.document(id)
self.db.queueDelete(doc)
all_docs = self.db.allDocs()
self.assertEqual(2, len(all_docs['rows']))
self.db.commit()
all_docs = self.db.allDocs()
self.assertEqual(0, len(all_docs['rows']))
def testReplicate(self):
repl_db = self.server.connectDatabase(self.db.name + 'repl')
doc_id = self.db.commitOne({'foo':123}, timestamp=True)[0]['id']
doc_v1 = self.db.document(doc_id)
#replicate
self.server.replicate(self.db.name, repl_db.name)
# wait for a few seconds to replication to be triggered.
time.sleep(1)
self.assertEqual(self.db.document(doc_id), repl_db.document(doc_id))
self.server.deleteDatabase(repl_db.name)
def testSlashInDBName(self):
"""
Slashes are a valid character in a database name, and are useful as it
creates a directory strucutre for the couch data files.
"""
db_name = 'wmcore/unittests'
try:
self.server.deleteDatabase(db_name)
except:
# Ignore this - the database shouldn't already exist
pass
db = self.server.createDatabase(db_name)
info = db.info()
assert info['db_name'] == db_name
db_name = 'wmcore/unittests'
db = self.server.connectDatabase(db_name)
info = db.info()
assert info['db_name'] == db_name
db = Database(db_name, url = os.environ["COUCHURL"])
info = db.info()
assert info['db_name'] == db_name
self.server.deleteDatabase(db_name)
def testInvalidName(self):
"""
Capitol letters are not allowed in database names.
"""
db_name = 'Not A Valid Name'
self.assertRaises(ValueError, self.server.createDatabase, db_name)
self.assertRaises(ValueError, self.server.deleteDatabase, db_name)
self.assertRaises(ValueError, self.server.connectDatabase, db_name)
self.assertRaises(ValueError, Database, db_name)
def testDocumentSerialisation(self):
"""
A document should be writable into the couchdb with a timestamp.
"""
d = Document()
d['foo'] = 'bar'
doc_info = self.db.commit(doc=d, timestamp=True)[0]
d_from_db = self.db.document(doc_info['id'])
self.assertEqual(d['foo'], d_from_db['foo'])
self.assertEqual(d['timestamp'], d_from_db['timestamp'])
def testAttachments(self):
"""
Test uploading attachments with and without checksumming
"""
doc = self.db.commitOne({'foo':'bar'}, timestamp=True)[0]
attachment1 = "Hello"
attachment2 = "How are you today?"
attachment3 = "I'm very well, thanks for asking"
attachment4 = "Lovely weather we're having"
attachment5 = "Goodbye"
keyhash = hashlib.md5()
keyhash.update(attachment5)
attachment5_md5 = keyhash.digest()
attachment5_md5 = base64.b64encode(attachment5_md5)
attachment6 = "Good day to you, sir!"
#TODO: add a binary attachment - e.g. tar.gz
doc = self.db.addAttachment(doc['id'], doc['rev'], attachment1)
doc = self.db.addAttachment(doc['id'], doc['rev'], attachment2, contentType="foo/bar")
doc = self.db.addAttachment(doc['id'], doc['rev'], attachment3, name="my_greeting")
doc = self.db.addAttachment(doc['id'], doc['rev'], attachment4, add_checksum=True)
doc = self.db.addAttachment(doc['id'], doc['rev'], attachment5, checksum=attachment5_md5)
self.assertRaises(CouchInternalServerError, self.db.addAttachment, doc['id'], doc['rev'], attachment6, checksum='123')
def testRevisionHandling(self):
# This test won't work from an existing database, conflicts will be preserved, so
# ruthlessly remove the databases to get a clean slate.
try:
self.server.deleteDatabase(self.db.name)
except CouchNotFoundError:
pass # Must have been deleted already
try:
self.server.deleteDatabase(self.db.name + 'repl')
except CouchNotFoundError:
pass # Must have been deleted already
# I'm going to create a conflict, so need a replica db
self.db = self.server.connectDatabase(self.db.name)
repl_db = self.server.connectDatabase(self.db.name + 'repl')
doc_id = self.db.commitOne({'foo':123}, timestamp=True)[0]['id']
doc_v1 = self.db.document(doc_id)
#replicate
self.server.replicate(self.db.name, repl_db.name)
time.sleep(1)
doc_v2 = self.db.document(doc_id)
doc_v2['bar'] = 456
doc_id_rev2 = self.db.commitOne(doc_v2)[0]
doc_v2 = self.db.document(doc_id)
#now update the replica
conflict_doc = repl_db.document(doc_id)
conflict_doc['bar'] = 101112
repl_db.commitOne(conflict_doc)
#replicate, creating the conflict
self.server.replicate(self.db.name, repl_db.name)
time.sleep(1)
conflict_view = {'map':"function(doc) {if(doc._conflicts) {emit(doc._conflicts, null);}}"}
data = repl_db.post('/%s/_temp_view' % repl_db.name, conflict_view)
# Should have one conflict in the repl database
self.assertEqual(data['total_rows'], 1)
# Should have no conflicts in the source database
self.assertEqual(self.db.post('/%s/_temp_view' % self.db.name, conflict_view)['total_rows'], 0)
self.assertTrue(repl_db.documentExists(data['rows'][0]['id'], rev=data['rows'][0]['key'][0]))
repl_db.delete_doc(data['rows'][0]['id'], rev=data['rows'][0]['key'][0])
data = repl_db.post('/%s/_temp_view' % repl_db.name, conflict_view)
self.assertEqual(data['total_rows'], 0)
self.server.deleteDatabase(repl_db.name)
#update it again
doc_v3 = self.db.document(doc_id)
doc_v3['baz'] = 789
doc_id_rev3 = self.db.commitOne(doc_v3)[0]
doc_v3 = self.db.document(doc_id)
#test that I can pull out an old revision
doc_v1_test = self.db.document(doc_id, rev=doc_v1['_rev'])
self.assertEqual(doc_v1, doc_v1_test)
#test that I can check a revision exists
self.assertTrue(self.db.documentExists(doc_id, rev=doc_v2['_rev']))
self.assertFalse(self.db.documentExists(doc_id, rev='1'+doc_v2['_rev']))
#why you shouldn't rely on rev
self.db.compact(blocking=True)
self.assertFalse(self.db.documentExists(doc_id, rev=doc_v1['_rev']))
self.assertFalse(self.db.documentExists(doc_id, rev=doc_v2['_rev']))
self.assertTrue(self.db.documentExists(doc_id, rev=doc_v3['_rev']))
def testCommit(self):
"""
Test queue and commit modes
"""
# try to commit 2 random docs
doc = {'foo':123, 'bar':456}
self.db.queue(doc)
self.db.queue(doc)
self.assertEqual(2, len(self.db.commit()))
# committing 2 docs with the same id will fail
self.db.queue(Document(id = "1", inputDict = {'foo':123, 'bar':456}))
self.db.queue(Document(id = "1", inputDict = {'foo':1234, 'bar':456}))
answer = self.db.commit()
self.assertEqual(2, len(answer))
self.assertEqual(answer[0]['ok'], True)
self.assertEqual(answer[1]['error'], 'conflict')
# all_or_nothing mode ignores conflicts
self.db.queue(Document(id = "2", inputDict = doc))
self.db.queue(Document(id = "2", inputDict = {'foo':1234, 'bar':456}))
answer = self.db.commit(all_or_nothing = True)
self.assertEqual(2, len(answer))
self.assertEqual(answer[0].get('error'), None)
self.assertEqual(answer[0].get('error'), None)
self.assertEqual(answer[0]['id'], '2')
self.assertEqual(answer[1]['id'], '2')
# callbacks can do stuff when conflicts arise
# this particular one just overwrites the document
def callback(db, data, result):
for doc in data['docs']:
if doc['_id'] == result['id']:
doc['_rev'] = db.document(doc['_id'])['_rev']
retval = db.commitOne(doc)
return retval[0]
self.db.queue(Document(id = "2", inputDict = {'foo':5, 'bar':6}))
answer = self.db.commit(callback = callback)
self.assertEqual(1, len(answer))
self.assertEqual(answer[0].get('error'), None)
updatedDoc = self.db.document('2')
self.assertEqual(updatedDoc['foo'], 5)
self.assertEqual(updatedDoc['bar'], 6)
return
def testUpdateHandler(self):
"""
Test that update function support works
"""
update_ddoc = {
'_id':'_design/foo',
'language': 'javascript',
'updates':{
"bump-counter" : 'function(doc, req) {if (!doc.counter) {doc.counter = 0};doc.counter += 1;return [doc,"bumped it!"];}',
}
}
self.db.commit(update_ddoc)
doc = {'foo': 123, 'counter': 0}
doc_id = self.db.commit(doc)[0]['id']
self.assertEqual("bumped it!", self.db.updateDocument(doc_id, 'foo', 'bump-counter'))
self.assertEqual(1, self.db.document(doc_id)['counter'])
def testList(self):
"""
Test list function works ok
"""
update_ddoc = {
'_id':'_design/foo',
'language': 'javascript',
'views' : {
'all' : {
'map' : 'function(doc) {emit(null, null) }'
},
},
'lists' : {
'errorinoutput' : 'function(doc, req) {send("A string with the word error in")}',
'malformed' : 'function(doc, req) {somethingtoraiseanerror}',
}
}
self.db.commit(update_ddoc)
# approriate errors raised
self.assertRaises(CouchNotFoundError, self.db.loadList, 'foo', 'error', 'view_doesnt_exist')
self.assertRaises(CouchInternalServerError, self.db.loadList, 'foo', 'malformed', 'all')
# error in list output string shouldn't raise an error
self.assertEqual(self.db.loadList('foo', 'errorinoutput', 'all'),
"A string with the word error in")
def testAllDocs(self):
"""
Test AllDocs with options
"""
self.db.queue(Document(id = "1", inputDict = {'foo':123, 'bar':456}))
self.db.queue(Document(id = "2", inputDict = {'foo':123, 'bar':456}))
self.db.queue(Document(id = "3", inputDict = {'foo':123, 'bar':456}))
self.db.commit()
self.assertEqual(3, len(self.db.allDocs()['rows']))
self.assertEqual(2, len(self.db.allDocs({'startkey': "2"})['rows']))
self.assertEqual(2, len(self.db.allDocs(keys = ["1", "3"])['rows']))
self.assertEqual(1, len(self.db.allDocs({'limit':1}, ["1", "3"])['rows']))
self.assertTrue('error' in self.db.allDocs(keys = ["1", "4"])['rows'][1])
def testUpdateBulkDocuments(self):
"""
Test AllDocs with options
"""
self.db.queue(Document(id="1", inputDict={'foo':123, 'bar':456}))
self.db.queue(Document(id="2", inputDict={'foo':123, 'bar':456}))
self.db.queue(Document(id="3", inputDict={'foo':123, 'bar':456}))
self.db.commit()
self.db.updateBulkDocumentsWithConflictHandle(["1", "2", "3"], {'foo': 333}, 2)
result = self.db.allDocs({"include_docs": True})['rows']
self.assertEqual(3, len(result))
for item in result:
self.assertEqual(333, item['doc']['foo'])
self.db.updateBulkDocumentsWithConflictHandle(["1", "2", "3"], {'foo': 222}, 10)
result = self.db.allDocs({"include_docs": True})['rows']
self.assertEqual(3, len(result))
for item in result:
self.assertEqual(222, item['doc']['foo'])
def testUpdateHandlerAndBulkUpdateProfile(self):
"""
Test that update function support works
"""
# for actual test increase the size value: For 10000 records, 96 sec vs 4 sec
size = 100
for i in range(size):
self.db.queue(Document(id="%s" % i, inputDict={'name':123, 'counter':0}))
update_doc = {
'_id':'_design/foo',
'language': 'javascript',
'updates':{
"change-counter" : """function(doc, req) { if (doc) { var data = JSON.parse(req.body);
for (var field in data) {doc.field = data.field;} return [doc, 'OK'];}}""",
}
}
self.db.commit(update_doc)
start = time.time()
for id in range(size):
doc_id = "%s" % id
self.db.updateDocument(doc_id, 'foo', 'change-counter', {'counter': 1}, useBody=True)
end = time.time()
print("update handler: %s sec" % (end - start))
start = time.time()
ids = []
for id in range(size):
doc_id = "%s" % id
ids.append(doc_id)
self.db.updateBulkDocumentsWithConflictHandle(ids, {'counter': 2}, 1000)
end = time.time()
print("bulk update: %s sec" % (end - start))
class DASCouchcache(Cache):
"""
Base DAS couchdb cache class based on couchdb, see
http://couchdb.apache.org/, The client API based on
http://wiki.apache.org/couchdb/Getting_started_with_Python
in particular we use couchdb-python library
http://couchdb-python.googlecode.com/
"""
def __init__(self, config):
Cache.__init__(self, config)
uri = config['couch_servers'] # in a future I may have several
self.logger = config['logger']
if not self.logger:
self.logger = DummyLogger()
self.limit = config['couch_lifetime']
self.uri = uri.replace('http://', '')
self.server = CouchServer(self.uri)
self.dbname = "das"
self.cdb = None # cached couch DB handler
self.future = 9999999999 # unreachable timestamp
self.logger.info('Init couchcache %s' % self.uri)
self.views = {
'query': {
'map':
"""
function(doc) {
if(doc.hash) {
emit([doc.hash, doc.expire], doc.results);
}
}"""
},
# 'incache': {'map': """
#function(doc) {
# if(doc.hash) {
# emit([doc.hash, doc.expire], null);
# }
#}"""
# },
}
self.adminviews = {
'system': {
'map':
"""
function(doc) {
if(doc.results.system) {
emit(doc.results.system, doc);
}
}"""
},
'cleaner': {
'map':
"""
function(doc) {
if(doc.expire) {
emit(doc.expire, doc);
}
}"""
},
'timer': {
'map':
"""
function(doc) {
if(doc.timestamp) {
emit(doc.timestamp, doc);
}
}"""
},
'all_queries': {
'map': """
function(doc) {
if (doc.query) {
emit(doc.query, null);
}
}""",
'reduce': """
function(keys, values) {
return null;
}"""
},
}
def connect(self, url):
"""
Connect to different Couch DB URL
"""
self.uri = url.replace('http://', '')
del self.server
self.server = CouchServer(self.uri)
def create_view(self, dbname, design, view_dict):
"""
Create new view in couch db.
"""
cdb = self.couchdb(dbname)
# check provided view_dict that it has all keys
for view, definition in view_dict.items():
if type(definition) is not dict:
msg = 'View "%s" has improper definition' % view
raise Exception(msg)
if 'map' not in definition:
msg = 'View "%s" does not have map'
raise Exception(msg)
view = dict(_id='_design/%s' % design,
language='javascript',
doctype='view',
views=view_dict)
cdb.commit(view)
def delete_view(self, dbname, design, view_name):
"""
Delete given view in couch db
"""
print("Delete view", dbname, design, view_name)
def dbinfo(self, dbname='das'):
"""
Provide couch db info
"""
cdb = self.couchdb(dbname)
if cdb:
self.logger.info(cdb.info())
else:
self.logger.warning("No '%s' found in couch db" % dbname)
if not cdb:
return "Unable to connect to %s" % dbname
return cdb.info()
def delete_cache(self, dbname=None, system=None):
"""
Delete either couchh db (dbname) or particular docs
for provided system, e.g. all sitedb docs.
"""
cdb = self.couchdb(dbname)
if cdb:
if system:
key = '"%s"' % system
options = {'key': key}
results = self.get_view('dasadmin', 'system', options)
for doc in results:
cdb.queuedelete(doc)
cdb.commit()
else:
self.server.deleteDatabase(dbname)
return
def couchdb(self, dbname):
"""
look up db in couch db server, if found give it back to user
"""
if self.cdb:
return self.cdb
couch_db_list = []
try:
couch_db_list = self.server.listDatabases()
except:
return None
if dbname not in couch_db_list:
self.logger.info("DASCouchcache::couchdb, create db %s" % dbname)
cdb = self.server.createDatabase(dbname)
self.create_view(self.dbname, 'dasviews', self.views)
self.create_view(self.dbname, 'dasadmin', self.adminviews)
else:
self.logger.info("DASCouchcache::couchdb, connect db %s" % dbname)
cdb = self.server.connectDatabase(dbname)
self.cdb = cdb
return cdb
def incache(self, query):
"""
Check if query exists in cache
"""
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
key = genkey(query)
#TODO:check how to query 1 result, I copied the way from get_from_cache
skey = ["%s" % key, timestamp()]
ekey = ["%s" % key, self.future]
options = {'startkey': skey, 'endkey': ekey}
# results = cdb.loadView('dasviews', 'incache', options)
results = cdb.loadView('dasviews', 'query', options)
try:
res = len(results['rows'])
except:
traceback.print_exc()
return
if res:
return True
return False
def get_from_cache(self, query, idx=0, limit=0, skey=None, order='asc'):
"""
Retreieve results from cache, otherwise return null.
"""
id = 0
idx = int(idx)
limit = long(limit)
stop = idx + limit # get upper bound for range
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
key = genkey(query)
skey = ["%s" % key, timestamp()]
ekey = ["%s" % key, self.future]
options = {'startkey': skey, 'endkey': ekey}
results = cdb.loadView('dasviews', 'query', options)
try:
res = [row['value'] for row in results['rows']]
for row in results['rows']:
row['id'] = id
if limit:
if id >= idx and id <= stop:
yield row
else:
yield row
id += 1
except:
traceback.print_exc()
return
if res:
self.logger.info("DASCouchcache::get_from_cache for %s" % query)
# if len(res) == 1:
# return res[0]
# return res
def update_cache(self, query, results, expire):
"""
Insert results into cache. We use bulk insert operation,
db.update over entire set, rather looping for every single
row and use db.create. The speed up is factor of 10
"""
if not expire:
raise Exception('Expire parameter is null')
self.logger.info("DASCouchcache::update_cache for %s" % query)
if not results:
return
dbname = self.dbname
viewlist = []
for key in self.views.keys():
viewlist.append("/%s/_design/dasviews/_view/%s" % (dbname, key))
cdb = self.couchdb(dbname)
self.clean_cache()
if not cdb:
if type(results) is list or \
type(results) is types.GeneratorType:
for row in results:
yield row
else:
yield results
return
if type(results) is list or \
type(results) is types.GeneratorType:
for row in results:
res = results2couch(query, row, expire)
cdb.queue(res, viewlist=viewlist)
yield row
else:
res = results2couch(query, results, expire)
yield results
cdb.queue(res, viewlist=viewlist)
cdb.commit(viewlist=viewlist)
def remove_from_cache(self, query):
"""
Delete query from cache
"""
self.logger.debug('DASCouchcache::remove_from_cache(%s)' \
% (query, ))
return
def get_view(self, design, view, options={}):
"""
Retreieve results from cache based on provided Couchcache view
"""
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
results = cdb.loadView(design, view, options)
res = [row['value'] for row in results['rows']]
if len(res) == 1:
return res[0]
return res
def list_views(self):
"""
Return a list of Couchcache views
"""
def clean_cache(self):
"""
Clean expired docs in couch db.
"""
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
skey = 0
ekey = timestamp()
options = {'startkey': skey, 'endkey': ekey}
results = cdb.loadView('dasadmin', 'cleaner', options)
ndocs = 0
for doc in results['rows']:
cdb.queueDelete(doc['value'])
ndocs += 1
self.logger.info("DASCouchcache::clean_couch, will remove %s doc's" \
% ndocs )
if not ndocs:
return
cdb.commit() # bulk delete
cdb.compact() # remove them permanently
def list_between(self, time_begin, time_end):
"""
Retreieve results from cache for time range
"""
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
skey = time_begin
ekey = time_end
options = {'startkey': skey, 'endkey': ekey}
results = cdb.loadView('dasadmin', 'timer', options)
try:
res = [row['value'] for row in results['rows']]
except:
traceback.print_exc()
return
if len(res) == 1:
return res[0]
return res
def list_queries_in(self, system, idx=0, limit=0):
"""
Retrieve results from cache for provided system, e.g. sitedb
"""
idx = int(idx)
limit = long(limit)
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
skey = system
ekey = system
options = {'startkey': skey, 'endkey': ekey}
results = cdb.loadView('dasadmin', 'system', options)
try:
res = [row['value'] for row in results['rows']]
except:
traceback.print_exc()
return
if len(res) == 1:
return res[0]
return res
def get_all_views(self, dbname=None):
"""
Method to get all degined views in couch db. The couch db doesn't have
a clear way to extract view documents. Instead we need to ask for
_all_docs and provide proper start/end-keys. Once we retrieve
_design docs, we loop over them and get the doc of particular view, e.g
http://localhost:5984/das/_design/dasviews
"""
if not dbname:
dbname = self.dbname
qqq = 'startkey=%22_design%2F%22&endkey=%22_design0%22'
host = 'http://' + self.uri
path = '/%s/_all_docs?%s' % (dbname, qqq)
kwds = {}
req = 'GET'
debug = 0
results = httplib_request(host, path, kwds, req, debug)
designdocs = json.loads(results)
results = {}
for item in designdocs['rows']:
doc = item['key']
# print "design:", doc
path = '/%s/%s' % (dbname, doc)
res = httplib_request(host, path, kwds, req, debug)
rdict = json.loads(res)
views = []
for view_name, view_dict in rdict['views'].items():
# print " view:", view_name
# print " map:", view_dict['map']
if 'reduce' in view_dict:
# print "reduce:", view_dict['reduce']
rdef = view_dict['reduce']
defrow = dict(map=view_dict['map'],
reduce=view_dict['reduce'])
else:
defrow = dict(map=view_dict['map'])
row = {'%s' % view_name: defrow}
views.append(row)
results[doc] = views
return results
def get_all_queries(self, idx=0, limit=0):
"""
Retreieve DAS queries from the cache.
"""
idx = int(idx)
limit = long(limit)
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
options = {}
results = cdb.loadView('dasadmin', 'all_queries', options)
try:
res = [row['value'] for row in results['rows']]
except:
traceback.print_exc()
return
if len(res) == 1:
return res[0]
return res
class RotatingDatabaseTest(unittest.TestCase):
def setUp(self):
self.couchURL = os.getenv("COUCHURL")
self.server = CouchServer(self.couchURL)
# Kill off any databases left over from previous runs
for db in [
db for db in self.server.listDatabases()
if db.startswith('rotdb_unittest_')
]:
try:
self.server.deleteDatabase(db)
except:
pass
# Create a database, drop an existing one first
testname = self.id().split('.')[-1].lower()
self.dbname = 'rotdb_unittest_%s' % testname
self.arcname = 'rotdb_unittest_%s_archive' % testname
self.seedname = 'rotdb_unittest_%s_seedcfg' % testname
# set a long value for times, tests do operations explicitly
self.timing = {
'archive': timedelta(seconds=1),
'expire': timedelta(seconds=2)
}
self.db = RotatingDatabase(dbname=self.dbname,
url=self.couchURL,
archivename=self.arcname,
timing=self.timing)
def tearDown(self):
testname = self.id().split('.')[-1].lower()
if sys.exc_info()[0] == None:
# This test has passed, clean up after it
to_go = [
db for db in self.server.listDatabases()
if db.startswith('rotdb_unittest_%s' % testname)
]
for dbname in to_go:
try:
self.server.deleteDatabase(dbname)
except CouchNotFoundError:
# db has already gone
pass
def testRotate(self):
"""
Test that rotation works
"""
start_name = self.db.name
self.db._rotate()
end_name = self.db.name
databases = [
db for db in self.server.listDatabases()
if db.startswith('rotdb_unittest_')
]
self.assertTrue(start_name in databases)
self.assertTrue(end_name in databases)
def testArchive(self):
"""
Test that archiving views works
"""
dummy_view = {
'_id': '_design/foo',
'language': 'javascript',
'views': {
'bar': {
'map': "function(doc) {if (doc.foo) {emit(doc.int, 1);}}",
'reduce': '_sum'
}
}
}
archive_view = {
'_id': '_design/foo',
'language': 'javascript',
'views': {
'bar': {
'map': "function(doc) {emit(doc.key, doc.value);}",
'reduce': '_sum'
}
}
}
seed_db = self.server.connectDatabase(self.seedname)
seed_db.commit(dummy_view)
# Need to have the timing long enough so the data isn't archived by accident
self.timing = {
'archive': timedelta(seconds=1000),
'expire': timedelta(seconds=2000)
}
self.db = RotatingDatabase(dbname=self.dbname,
url=self.couchURL,
views=['foo/bar'],
archivename=self.arcname,
timing=self.timing)
self.db.archive_db.commitOne(archive_view)
runs = 5
docs = 5
for run in range(runs):
for i in range(docs):
self.db.queue({'foo': 'bar', 'int': i, 'run': run})
self.db.commit()
self.db._rotate()
self.db._archive()
view_result = self.db.archive_db.loadView('foo', 'bar')
arch_sum = view_result['rows'][0]['value']
self.assertEqual(arch_sum, runs * docs)
def testExpire(self):
"""
Test that expiring databases works
"""
# rotate out the original db
self.db._rotate()
archived = self.db.archived_dbs()
self.assertEqual(1, len(archived),
'test not starting from clean state, bail!')
# Make sure the db has expired
sleep(2)
self.db._expire()
self.assertEqual(0, len(self.db.archived_dbs()))
self.assertFalse(archived[0] in self.server.listDatabases())
@attr("integration")
def testCycle(self):
"""
Test that committing data to different databases happens
This is a bit of a dodgy test - if timings go funny it will fail
"""
self.timing = {
'archive': timedelta(seconds=0.5),
'expire': timedelta(seconds=1)
}
self.db = RotatingDatabase(dbname=self.dbname,
url=self.couchURL,
archivename=self.arcname,
timing=self.timing)
my_name = self.db.name
self.db.commit({'foo': 'bar'})
sleep(5)
self.db.commit({'foo': 'bar'})
# the initial db should have expired by now
self.db.commit({'foo': 'bar'})
self.assertFalse(my_name in self.server.listDatabases(), "")
class DASCouchcache(Cache):
"""
Base DAS couchdb cache class based on couchdb, see
http://couchdb.apache.org/, The client API based on
http://wiki.apache.org/couchdb/Getting_started_with_Python
in particular we use couchdb-python library
http://couchdb-python.googlecode.com/
"""
def __init__(self, config):
Cache.__init__(self, config)
uri = config['couch_servers'] # in a future I may have several
self.logger = config['logger']
if not self.logger:
self.logger = DummyLogger()
self.limit = config['couch_lifetime']
self.uri = uri.replace('http://', '')
self.server = CouchServer(self.uri)
self.dbname = "das"
self.cdb = None # cached couch DB handler
self.future = 9999999999 # unreachable timestamp
self.logger.info('Init couchcache %s' % self.uri)
self.views = {
'query': {'map': """
function(doc) {
if(doc.hash) {
emit([doc.hash, doc.expire], doc.results);
}
}"""
},
# 'incache': {'map': """
#function(doc) {
# if(doc.hash) {
# emit([doc.hash, doc.expire], null);
# }
#}"""
# },
}
self.adminviews = {
'system' : {'map': """
function(doc) {
if(doc.results.system) {
emit(doc.results.system, doc);
}
}"""
},
'cleaner' : {'map': """
function(doc) {
if(doc.expire) {
emit(doc.expire, doc);
}
}"""
},
'timer' : {'map': """
function(doc) {
if(doc.timestamp) {
emit(doc.timestamp, doc);
}
}"""
},
'all_queries' : {'map': """
function(doc) {
if (doc.query) {
emit(doc.query, null);
}
}""",
'reduce' : """
function(keys, values) {
return null;
}"""
},
}
def connect(self, url):
"""
Connect to different Couch DB URL
"""
self.uri = url.replace('http://', '')
del self.server
self.server = CouchServer(self.uri)
def create_view(self, dbname, design, view_dict):
"""
Create new view in couch db.
"""
cdb = self.couchdb(dbname)
# check provided view_dict that it has all keys
for view, definition in view_dict.items():
if type(definition) is not dict:
msg = 'View "%s" has improper definition' % view
raise Exception(msg)
if 'map' not in definition:
msg = 'View "%s" does not have map'
raise Exception(msg)
view = dict(_id='_design/%s' % design, language='javascript',
doctype='view', views=view_dict)
cdb.commit(view)
def delete_view(self, dbname, design, view_name):
"""
Delete given view in couch db
"""
print("Delete view", dbname, design, view_name)
def dbinfo(self, dbname='das'):
"""
Provide couch db info
"""
cdb = self.couchdb(dbname)
if cdb:
self.logger.info(cdb.info())
else:
self.logger.warning("No '%s' found in couch db" % dbname)
if not cdb:
return "Unable to connect to %s" % dbname
return cdb.info()
def delete_cache(self, dbname=None, system=None):
"""
Delete either couchh db (dbname) or particular docs
for provided system, e.g. all sitedb docs.
"""
cdb = self.couchdb(dbname)
if cdb:
if system:
key = '"%s"' % system
options = {'key' : key}
results = self.get_view('dasadmin', 'system', options)
for doc in results:
cdb.queuedelete(doc)
cdb.commit()
else:
self.server.deleteDatabase(dbname)
return
def couchdb(self, dbname):
"""
look up db in couch db server, if found give it back to user
"""
if self.cdb:
return self.cdb
couch_db_list = []
try:
couch_db_list = self.server.listDatabases()
except:
return None
if dbname not in couch_db_list:
self.logger.info("DASCouchcache::couchdb, create db %s" % dbname)
cdb = self.server.createDatabase(dbname)
self.create_view(self.dbname, 'dasviews', self.views)
self.create_view(self.dbname, 'dasadmin', self.adminviews)
else:
self.logger.info("DASCouchcache::couchdb, connect db %s" % dbname)
cdb = self.server.connectDatabase(dbname)
self.cdb = cdb
return cdb
def incache(self, query):
"""
Check if query exists in cache
"""
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
key = genkey(query)
#TODO:check how to query 1 result, I copied the way from get_from_cache
skey = ["%s" % key, timestamp()]
ekey = ["%s" % key, self.future]
options = {'startkey': skey, 'endkey': ekey}
# results = cdb.loadView('dasviews', 'incache', options)
results = cdb.loadView('dasviews', 'query', options)
try:
res = len(results['rows'])
except:
traceback.print_exc()
return
if res:
return True
return False
def get_from_cache(self, query, idx=0, limit=0, skey=None, order='asc'):
"""
Retreieve results from cache, otherwise return null.
"""
id = 0
idx = int(idx)
limit = long(limit)
stop = idx + limit # get upper bound for range
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
key = genkey(query)
skey = ["%s" % key, timestamp()]
ekey = ["%s" % key, self.future]
options = {'startkey': skey, 'endkey': ekey}
results = cdb.loadView('dasviews', 'query', options)
try:
res = [row['value'] for row in results['rows']]
for row in results['rows']:
row['id'] = id
if limit:
if id >= idx and id <= stop:
yield row
else:
yield row
id += 1
except:
traceback.print_exc()
return
if res:
self.logger.info("DASCouchcache::get_from_cache for %s" % query)
# if len(res) == 1:
# return res[0]
# return res
def update_cache(self, query, results, expire):
"""
Insert results into cache. We use bulk insert operation,
db.update over entire set, rather looping for every single
row and use db.create. The speed up is factor of 10
"""
if not expire:
raise Exception('Expire parameter is null')
self.logger.info("DASCouchcache::update_cache for %s" % query)
if not results:
return
dbname = self.dbname
viewlist = []
for key in self.views.keys():
viewlist.append("/%s/_design/dasviews/_view/%s" % (dbname, key))
cdb = self.couchdb(dbname)
self.clean_cache()
if not cdb:
if type(results) is list or \
type(results) is types.GeneratorType:
for row in results:
yield row
else:
yield results
return
if type(results) is list or \
type(results) is types.GeneratorType:
for row in results:
res = results2couch(query, row, expire)
cdb.queue(res, viewlist=viewlist)
yield row
else:
res = results2couch(query, results, expire)
yield results
cdb.queue(res, viewlist=viewlist)
cdb.commit(viewlist=viewlist)
def remove_from_cache(self, query):
"""
Delete query from cache
"""
self.logger.debug('DASCouchcache::remove_from_cache(%s)' \
% (query, ))
return
def get_view(self, design, view, options={}):
"""
Retreieve results from cache based on provided Couchcache view
"""
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
results = cdb.loadView(design, view, options)
res = [row['value'] for row in results['rows']]
if len(res) == 1:
return res[0]
return res
def list_views(self):
"""
Return a list of Couchcache views
"""
def clean_cache(self):
"""
Clean expired docs in couch db.
"""
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
skey = 0
ekey = timestamp()
options = {'startkey': skey, 'endkey': ekey}
results = cdb.loadView('dasadmin', 'cleaner', options)
ndocs = 0
for doc in results['rows']:
cdb.queueDelete(doc['value'])
ndocs += 1
self.logger.info("DASCouchcache::clean_couch, will remove %s doc's" \
% ndocs )
if not ndocs:
return
cdb.commit() # bulk delete
cdb.compact() # remove them permanently
def list_between(self, time_begin, time_end):
"""
Retreieve results from cache for time range
"""
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
skey = time_begin
ekey = time_end
options = {'startkey': skey, 'endkey': ekey}
results = cdb.loadView('dasadmin', 'timer', options)
try:
res = [row['value'] for row in results['rows']]
except:
traceback.print_exc()
return
if len(res) == 1:
return res[0]
return res
def list_queries_in(self, system, idx=0, limit=0):
"""
Retrieve results from cache for provided system, e.g. sitedb
"""
idx = int(idx)
limit = long(limit)
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
skey = system
ekey = system
options = {'startkey': skey, 'endkey': ekey}
results = cdb.loadView('dasadmin', 'system', options)
try:
res = [row['value'] for row in results['rows']]
except:
traceback.print_exc()
return
if len(res) == 1:
return res[0]
return res
def get_all_views(self, dbname=None):
"""
Method to get all degined views in couch db. The couch db doesn't have
a clear way to extract view documents. Instead we need to ask for
_all_docs and provide proper start/end-keys. Once we retrieve
_design docs, we loop over them and get the doc of particular view, e.g
http://localhost:5984/das/_design/dasviews
"""
if not dbname:
dbname = self.dbname
qqq = 'startkey=%22_design%2F%22&endkey=%22_design0%22'
host = 'http://' + self.uri
path = '/%s/_all_docs?%s' % (dbname, qqq)
kwds = {}
req = 'GET'
debug = 0
results = httplib_request(host, path, kwds, req, debug)
designdocs = json.loads(results)
results = {}
for item in designdocs['rows']:
doc = item['key']
# print "design:", doc
path = '/%s/%s' % (dbname, doc)
res = httplib_request(host, path, kwds, req, debug)
rdict = json.loads(res)
views = []
for view_name, view_dict in rdict['views'].items():
# print " view:", view_name
# print " map:", view_dict['map']
if 'reduce' in view_dict:
# print "reduce:", view_dict['reduce']
rdef = view_dict['reduce']
defrow = dict(map=view_dict['map'],
reduce=view_dict['reduce'])
else:
defrow = dict(map=view_dict['map'])
row = {'%s' % view_name : defrow}
views.append(row)
results[doc] = views
return results
def get_all_queries(self, idx=0, limit=0):
"""
Retreieve DAS queries from the cache.
"""
idx = int(idx)
limit = long(limit)
dbname = self.dbname
cdb = self.couchdb(dbname)
if not cdb:
return
options = {}
results = cdb.loadView('dasadmin', 'all_queries', options)
try:
res = [row['value'] for row in results['rows']]
except:
traceback.print_exc()
return
if len(res) == 1:
return res[0]
return res
def configure(self):
server = CouchServer(self.config.JobStateMachine.couchurl)
dbname = 'JSM/JobHistory'
if dbname not in server.listDatabases():
server.createDatabase(dbname)
class ConfigCache(WMObject):
"""
_ConfigCache_
The class that handles the upload and download of configCache
artifacts from Couch
"""
def __init__(self,
dbURL,
couchDBName=None,
id=None,
rev=None,
usePYCurl=False,
ckey=None,
cert=None,
capath=None,
detail=True):
self.dbname = couchDBName
self.dburl = dbURL
self.detail = detail
try:
self.couchdb = CouchServer(self.dburl,
usePYCurl=usePYCurl,
ckey=ckey,
cert=cert,
capath=capath)
if self.dbname not in self.couchdb.listDatabases():
self.createDatabase()
self.database = self.couchdb.connectDatabase(self.dbname)
except Exception as ex:
msg = "Error connecting to couch: %s\n" % str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise ConfigCacheException(message=msg)
# local cache
self.docs_cache = DocumentCache(self.database, self.detail)
# UserGroup variables
self.group = None
self.owner = None
# Internal data structure
self.document = Document()
self.attachments = {}
self.document['type'] = "config"
self.document['description'] = {}
self.document['description']['config_label'] = None
self.document['description']['config_desc'] = None
if id != None:
self.document['_id'] = id
self.document['pset_tweak_details'] = None
self.document['info'] = None
self.document['config'] = None
return
def createDatabase(self):
"""
_createDatabase_
"""
database = self.couchdb.createDatabase(self.dbname)
database.commit()
return database
def connectUserGroup(self, groupname, username):
"""
_connectUserGroup_
"""
self.group = Group(name=groupname)
self.group.setCouch(self.dburl, self.dbname)
self.group.connect()
self.owner = makeUser(groupname,
username,
couchUrl=self.dburl,
couchDatabase=self.dbname)
return
def createUserGroup(self, groupname, username):
"""
_createUserGroup_
Create all the userGroup information
"""
self.createGroup(name=groupname)
self.createUser(username=username)
return
def createGroup(self, name):
"""
_createGroup_
Create Group for GroupUser
"""
self.group = Group(name=name)
self.group.setCouch(self.dburl, self.dbname)
self.group.connect()
self.group.create()
return
def setLabel(self, label):
"""
_setLabel_
Util to add a descriptive label to the configuration doc
"""
self.document['description']['config_label'] = label
def setDescription(self, desc):
"""
_setDescription_
Util to add a verbose description string to a configuration doc
"""
self.document['description']['config_desc'] = desc
@Decorators.requireGroup
def createUser(self, username):
self.owner = makeUser(self.group['name'],
username,
couchUrl=self.dburl,
couchDatabase=self.dbname)
self.owner.create()
self.owner.ownThis(self.document)
return
@Decorators.requireGroup
@Decorators.requireUser
def save(self):
"""
_save_
Save yourself! Save your internal document.
"""
rawResults = self.database.commit(doc=self.document)
# We should only be committing one document at a time
# if not, get the last one.
try:
commitResults = rawResults[-1]
self.document["_rev"] = commitResults.get('rev')
self.document["_id"] = commitResults.get('id')
except KeyError as ex:
msg = "Document returned from couch without ID or Revision\n"
msg += "Document probably bad\n"
msg += str(ex)
logging.error(msg)
raise ConfigCacheException(message=msg)
# Now do the attachments
for attachName in self.attachments:
self.saveAttachment(name=attachName,
attachment=self.attachments[attachName])
return
def saveAttachment(self, name, attachment):
"""
_saveAttachment_
Save an attachment to the document
"""
retval = self.database.addAttachment(self.document["_id"],
self.document["_rev"], attachment,
name)
if retval.get('ok', False) != True:
# Then we have a problem
msg = "Adding an attachment to document failed\n"
msg += str(retval)
msg += "ID: %s, Rev: %s" % (self.document["_id"],
self.document["_rev"])
logging.error(msg)
raise ConfigCacheException(msg)
self.document["_rev"] = retval['rev']
self.document["_id"] = retval['id']
self.attachments[name] = attachment
return
def loadDocument(self, configID):
"""
_loadDocument_
Load a document from the document cache given its couchID
"""
self.document = self.docs_cache[configID]
def loadByID(self, configID):
"""
_loadByID_
Load a document from the server given its couchID
"""
try:
self.document = self.database.document(id=configID)
if 'owner' in self.document.keys():
self.connectUserGroup(
groupname=self.document['owner'].get('group', None),
username=self.document['owner'].get('user', None))
if '_attachments' in self.document.keys():
# Then we need to load the attachments
for key in self.document['_attachments'].keys():
self.loadAttachment(name=key)
except CouchNotFoundError as ex:
msg = "Document with id %s not found in couch\n" % (configID)
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise ConfigCacheException(message=msg)
except Exception as ex:
msg = "Error loading document from couch\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise ConfigCacheException(message=msg)
return
def loadAttachment(self, name, overwrite=True):
"""
_loadAttachment_
Load an attachment from the database and put it somewhere useful
"""
attach = self.database.getAttachment(self.document["_id"], name)
if not overwrite:
if name in self.attachments.keys():
logging.info("Attachment already exists, so we're skipping")
return
self.attachments[name] = attach
return
def loadByView(self, view, value):
"""
_loadByView_
Underlying code to load views
"""
viewRes = self.database.loadView('ConfigCache', view, {}, [value])
if len(viewRes['rows']) == 0:
# Then we have a problem
logging.error("Unable to load using view %s and value %s" %
(view, str(value)))
self.unwrapView(viewRes)
self.loadByID(self.document["_id"])
return
def saveConfigToDisk(self, targetFile):
"""
_saveConfigToDisk_
Make sure we can save our config file to disk
"""
config = self.getConfig()
if not config:
return
# Write to a file
f = open(targetFile, 'w')
f.write(config)
f.close()
return
def load(self):
"""
_load_
Figure out how to load
"""
if self.document.get("_id", None) != None:
# Then we should load by ID
self.loadByID(self.document["_id"])
return
# Otherwise we have to load by view
if not self.document.get('md5_hash', None) == None:
# Then we have an md5_hash
self.loadByView(view='config_by_md5hash',
value=self.document['md5_hash'])
# TODO: Add more views as they become available.
#elif not self.owner == None:
# Then we have an owner
#self.loadByView(view = 'config_by_owner', value = self.owner['name'])
def unwrapView(self, view):
"""
_unwrapView_
Move view information into the main document
"""
self.document["_id"] = view['rows'][0].get('id')
self.document["_rev"] = view['rows'][0].get('value').get('_rev')
def setPSetTweaks(self, PSetTweak):
"""
_setPSetTweaks_
Set the PSet tweak details for the config.
"""
self.document['pset_tweak_details'] = PSetTweak
return
def getPSetTweaks(self):
"""
_getPSetTweaks_
Retrieve the PSet tweak details.
"""
return self.document['pset_tweak_details']
def getOutputModuleInfo(self):
"""
_getOutputModuleInfo_
Retrieve the dataset information for the config in the ConfigCache.
"""
psetTweaks = self.getPSetTweaks()
if not 'process' in psetTweaks.keys():
raise ConfigCacheException(
"Could not find process field in PSet while getting output modules!"
)
try:
outputModuleNames = psetTweaks["process"]["outputModules_"]
except KeyError as ex:
msg = "Could not find outputModules_ in psetTweaks['process'] while getting output modules.\n"
msg += str(ex)
logging.error(msg)
raise ConfigCacheException(msg)
results = {}
for outputModuleName in outputModuleNames:
try:
outModule = psetTweaks["process"][outputModuleName]
except KeyError:
msg = "Could not find outputModule %s in psetTweaks['process']" % outputModuleName
logging.error(msg)
raise ConfigCacheException(msg)
dataset = outModule.get("dataset", None)
if dataset:
results[outputModuleName] = {
"dataTier": outModule["dataset"]["dataTier"],
"filterName": outModule["dataset"]["filterName"]
}
else:
results[outputModuleName] = {
"dataTier": None,
"filterName": None
}
return results
def addConfig(self, newConfig, psetHash=None):
"""
_addConfig_
"""
# The newConfig parameter is a URL suitable for passing to urlopen.
configString = urllib.urlopen(newConfig).read(-1)
configMD5 = hashlib.md5(configString).hexdigest()
self.document['md5_hash'] = configMD5
self.document['pset_hash'] = psetHash
self.attachments['configFile'] = configString
return
def getConfig(self):
"""
_getConfig_
Get the currently active config
"""
return self.attachments.get('configFile', None)
def getCouchID(self):
"""
_getCouchID_
Return the document's couchID
"""
return self.document["_id"]
def getCouchRev(self):
"""
_getCouchRev_
Return the document's couchRevision
"""
return self.document["_rev"]
@Decorators.requireGroup
@Decorators.requireUser
def delete(self):
"""
_delete_
Deletes the document with the current docid
"""
if not self.document["_id"]:
logging.error("Attempted to delete with no couch ID")
# TODO: Delete without loading first
try:
self.database.queueDelete(self.document)
self.database.commit()
except Exception as ex:
msg = "Error in deleting document from couch"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise ConfigCacheException(message=msg)
return
def getIDFromLabel(self, label):
"""
_getIDFromLabel_
Retrieve the ID of a config given it's label.
"""
results = self.database.loadView("ConfigCache", "config_by_label", {
"startkey": label,
"limit": 1
})
if results["rows"][0]["key"] == label:
return results["rows"][0]["value"]
return None
def listAllConfigsByLabel(self):
"""
_listAllConfigsByLabel_
Retrieve a list of all the configs in the config cache. This is
returned in the form of a dictionary that is keyed by label.
"""
configs = {}
results = self.database.loadView("ConfigCache", "config_by_label")
for result in results["rows"]:
configs[result["key"]] = result["value"]
return configs
def __str__(self):
"""
Make something printable
"""
return self.document.__str__()
def validate(self, configID):
try:
#TODO: need to change to DataCache
#self.loadDocument(configID = configID)
self.loadByID(configID=configID)
except Exception as ex:
raise ConfigCacheException(
"Failure to load ConfigCache while validating workload: %s" %
str(ex))
if self.detail:
duplicateCheck = {}
try:
outputModuleInfo = self.getOutputModuleInfo()
except Exception as ex:
# Something's gone wrong with trying to open the configCache
msg = "Error in getting output modules from ConfigCache during workload validation. Check ConfigCache formatting!"
raise ConfigCacheException("%s: %s" % (msg, str(ex)))
for outputModule in outputModuleInfo.values():
dataTier = outputModule.get('dataTier', None)
filterName = outputModule.get('filterName', None)
if not dataTier:
raise ConfigCacheException("No DataTier in output module.")
# Add dataTier to duplicate dictionary
if not dataTier in duplicateCheck.keys():
duplicateCheck[dataTier] = []
if filterName in duplicateCheck[dataTier]:
# Then we've seen this combination before
raise ConfigCacheException(
"Duplicate dataTier/filterName combination.")
else:
duplicateCheck[dataTier].append(filterName)
return outputModuleInfo
else:
return True
class LogDBBackend(object):
"""
Represents persistent storage for LogDB
"""
def __init__(self, db_url, db_name, identifier, thread_name, **kwds):
self.db_url = db_url
self.server = CouchServer(db_url)
self.db_name = db_name
self.dbid = identifier
self.thread_name = thread_name
self.agent = kwds.get('agent', 0)
create = kwds.get('create', False)
size = kwds.get('size', 10000)
self.db = self.server.connectDatabase(db_name,
create=create,
size=size)
self.design = kwds.get('design', 'LogDB') # name of design document
self.view = kwds.get('view',
'requests') # name of view to look-up requests
self.tsview = kwds.get('tsview',
'tstamp') # name of tsview to look-up requests
if create:
uri = '/%s/_design/%s' % (db_name, self.design)
data = design_doc()
try:
# insert design doc, if fails due to conflict continue
# conflict may happen due to concurrent client connection who
# created first this doc
self.db.put(uri, data)
except CouchConflictError:
pass
def deleteDatabase(self):
"""Delete back-end database"""
if self.db_name in self.server.listDatabases():
self.server.deleteDatabase(self.db_name)
def check(self, request, mtype=None):
"""Check that given request name is valid"""
# TODO: we may add some logic to check request name, etc.
if not request:
raise LogDBError("Request name is empty")
if mtype and mtype not in LOGDB_MSG_TYPES:
raise LogDBError("Unsupported message type: '%s', supported types %s" \
% (mtype, LOGDB_MSG_TYPES))
def docid(self, request, mtype):
"""Generate doc id, we use double dash to avoid dashes from thread names"""
return gen_hash('--'.join(
(request, self.dbid, self.thread_name, mtype)))
def prefix(self, mtype):
"""Generate agent specific prefix for given message type"""
if self.agent:
# we add prefix for agent messages, all others will not have this index
mtype = 'agent-%s' % mtype
return mtype
def agent_update(self, request, msg='', mtype="info"):
"""Update agent info in LogDB for given request"""
self.check(request, mtype)
mtype = self.prefix(mtype)
rec = {"ts": tstamp(), "msg": msg}
doc = {
"_id": self.docid(request, mtype),
"messages": [rec],
"request": request,
"identifier": self.dbid,
"thr": self.thread_name,
"type": mtype
}
try:
exist_doc = self.db.document(doc["_id"])
doc["_rev"] = exist_doc["_rev"]
except CouchNotFoundError:
# this means document is not exist so we will just insert
pass
finally:
res = self.db.commitOne(doc)
return res
def user_update(self, request, msg, mtype='comment'):
"""Update user info in LogDB for given request"""
rec = {"ts": tstamp(), "msg": msg}
doc = {
"_id": self.docid(request, mtype),
"messages": [rec],
"request": request,
"identifier": self.dbid,
"thr": self.thread_name,
"type": mtype
}
try:
exist_doc = self.db.document(doc["_id"])
doc["_rev"] = exist_doc["_rev"]
doc["messages"] += exist_doc["messages"]
except CouchNotFoundError:
# this means document is not exist so we will just insert
pass
finally:
res = self.db.commitOne(doc)
return res
def get(self, request, mtype=None, detail=True):
"""Retrieve all entries from LogDB for given request"""
self.check(request, mtype)
spec = {'request': request, 'reduce': False}
if mtype:
spec.update({'type': mtype})
if detail:
spec.update({'include_docs': True})
docs = self.db.loadView(self.design, self.view, spec)
return docs
def get_all_requests(self):
"""Retrieve all entries from LogDB"""
spec = {'reduce': True, 'group_level': 1}
docs = self.db.loadView(self.design, self.view, spec)
return docs
def delete(self, request):
"""Delete entry in LogDB for given request"""
self.check(request)
docs = self.get(request, detail=False)
ids = [r['id'] for r in docs.get('rows', [])]
res = self.db.bulkDeleteByIDs(ids)
return res
def cleanup(self, thr):
"""
Clean-up docs older then given threshold (thr should be specified in seconds).
This is done via tstamp view end endkey, e.g.
curl "http://127.0.0.1:5984/logdb/_design/LogDB/_view/tstamp?endkey=1427912282"
"""
tstamp = round(time.time() - thr)
docs = self.db.allDocs() # may need another view to look-up old docs
spec = {'endkey': tstamp, 'reduce': False}
docs = self.db.loadView(self.design, self.tsview, spec)
ids = [d['id'] for d in docs.get('rows', [])]
self.db.bulkDeleteByIDs(ids)
