class AuxBaseAPI(RESTEntity):
"""
Base class for Aux db RESTEntry which contains get, post method
"""
def __init__(self, app, api, config, mount):
RESTEntity.__init__(self, app, api, config, mount)
# CouchDB auxiliary database name
self.reqmgr_aux_db = api.db_handler.get_db(config.couch_reqmgr_aux_db)
self.reqmgr_aux_db_service = RequestDBReader(self.reqmgr_aux_db,
couchapp="ReqMgrAux")
self.setName()
def setName(self):
"Sets the document name"
raise NotImplementedError(
"Couch document id(name) should be specified. i.e. self.name='software'"
)
def validate(self, apiobj, method, api, param, safe):
args_length = len(param.args)
if args_length == 1:
safe.kwargs["subName"] = param.args.pop(0)
return
return
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
def get(self, subName=None):
"""
Return entire self.name document
subName is subcategory of document which is added as postfix string
"""
try:
if subName:
if subName.lower() == "all_docs":
return rows(self._getAllDocs())
else:
docName = "%s_%s" % (self.name, subName)
else:
docName = self.name
sw = self.reqmgr_aux_db.document(docName)
del sw["_id"]
del sw["_rev"]
except CouchNotFoundError:
raise NoSuchInstance
return rows([sw])
def _getAllDocs(self):
"""
Return all the documents under a given document type
"""
try:
option = {"include_docs": True}
allDocs = self.reqmgr_aux_db_service.getRequestByCouchView(
"byconfig", option, [self.name])
except CouchError as ex:
msg = "ERROR: Failed to fetch ALL_DOCS for ConfigType: %s. Reason: %s" % (
self.name, str(ex))
cherrypy.log(msg)
raise cherrypy.HTTPError(404, msg)
return allDocs.values()
@restcall(formats=[('application/json', JSONFormat())])
def post(self, subName=None):
"""
Inserts a new document into the database
"""
data = cherrypy.request.body.read()
if not data:
raise MissingPostData()
else:
doc = json.loads(data)
if subName:
docName = "%s_%s" % (self.name, subName)
else:
docName = self.name
doc["ConfigType"] = self.name
doc = Document(docName, doc)
result = self.reqmgr_aux_db.commitOne(doc)
return result
@restcall(formats=[('text/plain', PrettyJSONFormat()),
('application/json', JSONFormat())])
def put(self, subName=None):
"""
Update document for the given self.name and subName.
It assumes the client has provided the entire entity, i.e., the old
content gets completely replaced by the new one.
Given that the each couch document contains a revision number, these PUT calls
are not going to be idempotent.
"""
data = cherrypy.request.body.read()
if not data:
raise MissingPostData()
else:
propertyDict = json.loads(data)
result = None
if subName:
docName = "%s_%s" % (self.name, subName)
else:
docName = self.name
try:
existDoc = self.reqmgr_aux_db.document(docName)
# replace original document
newDoc = Document(existDoc['_id'],
inputDict={
'_rev': existDoc['_rev'],
'ConfigType': existDoc['ConfigType']
})
newDoc.update(propertyDict)
result = self.reqmgr_aux_db.commitOne(newDoc)
except CouchNotFoundError:
cherrypy.log("Document %s not found. Creating one." % docName)
doc = Document(docName, propertyDict)
doc.update({'ConfigType': self.name})
result = self.reqmgr_aux_db.commitOne(doc)
return result
@restcall(formats=[('application/json', JSONFormat())])
def delete(self, subName):
"""
Delete a document from ReqMgrAux
"""
docName = "%s_%s" % (self.name, subName)
try:
res = self.reqmgr_aux_db.delete_doc(docName)
except (CouchError, CouchNotFoundError) as ex:
msg = "ERROR: failed to delete document: %s\nReason: %s" % (
docName, str(ex))
cherrypy.log(msg)
res = None
return res
def main():
"""
It will either delete docs in couchdb for the workflow you
have provided or it will loop over the final (or almost final)
states and ask for your permission to delete them.
"""
wfName = sys.argv[1] if len(sys.argv) == 2 else []
if 'WMAGENT_CONFIG' not in os.environ:
os.environ[
'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
# Instantiating central services (couch stuff)
# print "Central Couch URL : %s" % config.WorkloadSummary.couchurl
# print "Central ReqMgr URL : %s\n" % config.AnalyticsDataCollector.centralRequestDBURL
wfDBReader = RequestDBReader(
config.AnalyticsDataCollector.centralRequestDBURL,
couchapp=config.AnalyticsDataCollector.RequestCouchApp)
# Central services
wqBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
wqInboxDB = Database('workqueue_inbox', config.WorkloadSummary.couchurl)
# Local services
localWQBackend = WorkQueueBackend(config.WorkQueueManager.couchurl,
db_name="workqueue_inbox")
localWQInboxDB = Database('workqueue', config.WorkQueueManager.couchurl)
statusList = [
"failed", "epic-FAILED", "completed", "closed-out", "announced",
"aborted", "aborted-completed", "rejected", "normal-archived",
"aborted-archived", "rejected-archived"
]
for stat in final_status:
# retrieve list of workflows in each status
if not wfName:
# options = {'include_docs': False}
date_range = {
'startkey': [2015, 5, 15, 0, 0, 0],
'endkey': [2015, 5, 26, 0, 0, 0]
}
# finalWfs = wfDBReader.getRequestByCouchView("bydate", options, date_range)
tempWfs = wfDBReader.getRequestByCouchView("bydate", date_range)
#print "Found %d wfs in status: %s" %(len(finalWfs), stat)
finalWfs = []
for wf, content in tempWfs.iteritems():
if content['RequestStatus'] in statusList:
finalWfs.append(wf)
print "Found %d wfs in not in active state" % len(finalWfs)
else:
finalWfs = [wfName]
tempWfs = wfDBReader.getRequestByNames(wfName, True)
print "Checking %s with status '%s'." % (
wfName, tempWfs[wfName]['RequestStatus'])
wqDocs, wqInboxDocs = [], []
localWQDocs, localWQInboxDocs = [], []
for counter, wf in enumerate(finalWfs):
if counter % 100 == 0:
print "%d wfs queried ..." % counter
# check whether there are workqueue docs
wqDocIDs = wqBackend.getElements(WorkflowName=wf)
if wqDocIDs:
print "Found %d workqueue docs for %s, status %s" % (
len(wqDocIDs), wf, tempWfs[wf]['RequestStatus'])
print wqDocIDs
wqDocs.append(wqDocIDs)
# check whether there are workqueue_inbox docs
if wqInboxDB.documentExists(wf):
print "Found workqueue_inbox doc for %s, status %s" % (
wf, tempWfs[wf]['RequestStatus'])
# then retrieve the document
wqInboxDoc = wqInboxDB.document(wf)
wqInboxDocs.append(wqInboxDoc)
# check local queue
wqDocIDs = localWQBackend.getElements(WorkflowName=wf)
if wqDocIDs:
print "Found %d local workqueue docs for %s, status %s" % (
len(wqDocIDs), wf, tempWfs[wf]['RequestStatus'])
print wqDocIDs
localWQDocs.append(wqDocIDs)
if localWQInboxDB.documentExists(wf):
print "Found local workqueue_inbox doc for %s, status %s" % (
wf, tempWfs[wf]['RequestStatus'])
wqInboxDoc = localWQInboxDB.document(wf)
print wqInboxDoc
localWQInboxDocs.append(wqInboxDoc)
# TODO TODO TODO for the moment only deletes for a specific workflow
if wfName:
var = raw_input("\nCan we delete all these documents (Y/N)? ")
if var == "Y":
# deletes workqueue_inbox doc
if wqInboxDoc:
print "Deleting workqueue_inbox id %s and %s" % (
wqInboxDoc['_id'], wqInboxDoc['_rev'])
wqInboxDB.delete_doc(wqInboxDoc['_id'], wqInboxDoc['_rev'])
# deletes workqueue docs
if wqDocIDs:
print "Deleting workqueue docs %s" % wqDocIDs
wqBackend.deleteElements(
*[x for x in wqDocIDs if x['RequestName'] in wfName])
else:
print "You are the boss, aborting it ...\n"
def main():
"""
It will either delete docs in couchdb for the workflow you
have provided or it will loop over the final (or almost final)
states and ask for your permission to delete them.
"""
wfName = sys.argv[1] if len(sys.argv) == 2 else []
if 'WMAGENT_CONFIG' not in os.environ:
os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
# Instantiating central services (couch stuff)
# print "Central Couch URL : %s" % config.WorkloadSummary.couchurl
# print "Central ReqMgr URL : %s\n" % config.AnalyticsDataCollector.centralRequestDBURL
wfDBReader = RequestDBReader(config.AnalyticsDataCollector.centralRequestDBURL,
couchapp = config.AnalyticsDataCollector.RequestCouchApp)
# Central services
wqBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
wqInboxDB = Database('workqueue_inbox', config.WorkloadSummary.couchurl)
# Local services
localWQBackend = WorkQueueBackend(config.WorkQueueManager.couchurl, db_name = "workqueue_inbox")
localWQInboxDB = Database('workqueue', config.WorkQueueManager.couchurl)
statusList = ["failed", "epic-FAILED", "completed", "closed-out",
"announced", "aborted", "aborted-completed", "rejected",
"normal-archived", "aborted-archived", "rejected-archived"]
for stat in final_status:
# retrieve list of workflows in each status
if not wfName:
# options = {'include_docs': False}
date_range = {'startkey': [2015,5,15,0,0,0], 'endkey': [2015,5,26,0,0,0]}
# finalWfs = wfDBReader.getRequestByCouchView("bydate", options, date_range)
tempWfs = wfDBReader.getRequestByCouchView("bydate", date_range)
#print "Found %d wfs in status: %s" %(len(finalWfs), stat)
finalWfs = []
for wf, content in tempWfs.iteritems():
if content['RequestStatus'] in statusList:
finalWfs.append(wf)
print "Found %d wfs in not in active state" % len(finalWfs)
else:
finalWfs = [wfName]
tempWfs = wfDBReader.getRequestByNames(wfName, True)
print "Checking %s with status '%s'." % (wfName, tempWfs[wfName]['RequestStatus'])
wqDocs, wqInboxDocs = [], []
localWQDocs, localWQInboxDocs = [], []
for counter, wf in enumerate(finalWfs):
if counter % 100 == 0:
print "%d wfs queried ..." % counter
# check whether there are workqueue docs
wqDocIDs = wqBackend.getElements(WorkflowName = wf)
if wqDocIDs:
print "Found %d workqueue docs for %s, status %s" % (len(wqDocIDs), wf, tempWfs[wf]['RequestStatus'])
print wqDocIDs
wqDocs.append(wqDocIDs)
# check whether there are workqueue_inbox docs
if wqInboxDB.documentExists(wf):
print "Found workqueue_inbox doc for %s, status %s" % (wf, tempWfs[wf]['RequestStatus'])
# then retrieve the document
wqInboxDoc = wqInboxDB.document(wf)
wqInboxDocs.append(wqInboxDoc)
# check local queue
wqDocIDs = localWQBackend.getElements(WorkflowName = wf)
if wqDocIDs:
print "Found %d local workqueue docs for %s, status %s" % (len(wqDocIDs), wf, tempWfs[wf]['RequestStatus'])
print wqDocIDs
localWQDocs.append(wqDocIDs)
if localWQInboxDB.documentExists(wf):
print "Found local workqueue_inbox doc for %s, status %s" % (wf, tempWfs[wf]['RequestStatus'])
wqInboxDoc = localWQInboxDB.document(wf)
print wqInboxDoc
localWQInboxDocs.append(wqInboxDoc)
# TODO TODO TODO for the moment only deletes for a specific workflow
if wfName:
var = raw_input("\nCan we delete all these documents (Y/N)? ")
if var == "Y":
# deletes workqueue_inbox doc
if wqInboxDoc:
print "Deleting workqueue_inbox id %s and %s" % (wqInboxDoc['_id'], wqInboxDoc['_rev'])
wqInboxDB.delete_doc(wqInboxDoc['_id'], wqInboxDoc['_rev'])
# deletes workqueue docs
if wqDocIDs:
print "Deleting workqueue docs %s" % wqDocIDs
wqBackend.deleteElements(*[x for x in wqDocIDs if x['RequestName'] in wfName])
else:
print "You are the boss, aborting it ...\n"
