class AuxBaseAPI(RESTEntity): """ Base class for Aux db RESTEntry which contains get, post method """ def __init__(self, app, api, config, mount): RESTEntity.__init__(self, app, api, config, mount) # CouchDB auxiliary database name self.reqmgr_aux_db = api.db_handler.get_db(config.couch_reqmgr_aux_db) self.reqmgr_aux_db_service = RequestDBReader(self.reqmgr_aux_db, couchapp="ReqMgrAux") self.setName() def setName(self): "Sets the document name" raise NotImplementedError( "Couch document id(name) should be specified. i.e. self.name='software'" ) def validate(self, apiobj, method, api, param, safe): args_length = len(param.args) if args_length == 1: safe.kwargs["subName"] = param.args.pop(0) return return @restcall(formats=[('text/plain', PrettyJSONFormat()), ('application/json', JSONFormat())]) def get(self, subName=None): """ Return entire self.name document subName is subcategory of document which is added as postfix string """ try: if subName: if subName.lower() == "all_docs": return rows(self._getAllDocs()) else: docName = "%s_%s" % (self.name, subName) else: docName = self.name sw = self.reqmgr_aux_db.document(docName) del sw["_id"] del sw["_rev"] except CouchNotFoundError: raise NoSuchInstance return rows([sw]) def _getAllDocs(self): """ Return all the documents under a given document type """ try: option = {"include_docs": True} allDocs = self.reqmgr_aux_db_service.getRequestByCouchView( "byconfig", option, [self.name]) except CouchError as ex: msg = "ERROR: Failed to fetch ALL_DOCS for ConfigType: %s. Reason: %s" % ( self.name, str(ex)) cherrypy.log(msg) raise cherrypy.HTTPError(404, msg) return allDocs.values() @restcall(formats=[('application/json', JSONFormat())]) def post(self, subName=None): """ Inserts a new document into the database """ data = cherrypy.request.body.read() if not data: raise MissingPostData() else: doc = json.loads(data) if subName: docName = "%s_%s" % (self.name, subName) else: docName = self.name doc["ConfigType"] = self.name doc = Document(docName, doc) result = self.reqmgr_aux_db.commitOne(doc) return result @restcall(formats=[('text/plain', PrettyJSONFormat()), ('application/json', JSONFormat())]) def put(self, subName=None): """ Update document for the given self.name and subName. It assumes the client has provided the entire entity, i.e., the old content gets completely replaced by the new one. Given that the each couch document contains a revision number, these PUT calls are not going to be idempotent. """ data = cherrypy.request.body.read() if not data: raise MissingPostData() else: propertyDict = json.loads(data) result = None if subName: docName = "%s_%s" % (self.name, subName) else: docName = self.name try: existDoc = self.reqmgr_aux_db.document(docName) # replace original document newDoc = Document(existDoc['_id'], inputDict={ '_rev': existDoc['_rev'], 'ConfigType': existDoc['ConfigType'] }) newDoc.update(propertyDict) result = self.reqmgr_aux_db.commitOne(newDoc) except CouchNotFoundError: cherrypy.log("Document %s not found. Creating one." % docName) doc = Document(docName, propertyDict) doc.update({'ConfigType': self.name}) result = self.reqmgr_aux_db.commitOne(doc) return result @restcall(formats=[('application/json', JSONFormat())]) def delete(self, subName): """ Delete a document from ReqMgrAux """ docName = "%s_%s" % (self.name, subName) try: res = self.reqmgr_aux_db.delete_doc(docName) except (CouchError, CouchNotFoundError) as ex: msg = "ERROR: failed to delete document: %s\nReason: %s" % ( docName, str(ex)) cherrypy.log(msg) res = None return res
def main(): """ It will either delete docs in couchdb for the workflow you have provided or it will loop over the final (or almost final) states and ask for your permission to delete them. """ wfName = sys.argv[1] if len(sys.argv) == 2 else [] if 'WMAGENT_CONFIG' not in os.environ: os.environ[ 'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py' config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"]) # Instantiating central services (couch stuff) # print "Central Couch URL : %s" % config.WorkloadSummary.couchurl # print "Central ReqMgr URL : %s\n" % config.AnalyticsDataCollector.centralRequestDBURL wfDBReader = RequestDBReader( config.AnalyticsDataCollector.centralRequestDBURL, couchapp=config.AnalyticsDataCollector.RequestCouchApp) # Central services wqBackend = WorkQueueBackend(config.WorkloadSummary.couchurl) wqInboxDB = Database('workqueue_inbox', config.WorkloadSummary.couchurl) # Local services localWQBackend = WorkQueueBackend(config.WorkQueueManager.couchurl, db_name="workqueue_inbox") localWQInboxDB = Database('workqueue', config.WorkQueueManager.couchurl) statusList = [ "failed", "epic-FAILED", "completed", "closed-out", "announced", "aborted", "aborted-completed", "rejected", "normal-archived", "aborted-archived", "rejected-archived" ] for stat in final_status: # retrieve list of workflows in each status if not wfName: # options = {'include_docs': False} date_range = { 'startkey': [2015, 5, 15, 0, 0, 0], 'endkey': [2015, 5, 26, 0, 0, 0] } # finalWfs = wfDBReader.getRequestByCouchView("bydate", options, date_range) tempWfs = wfDBReader.getRequestByCouchView("bydate", date_range) #print "Found %d wfs in status: %s" %(len(finalWfs), stat) finalWfs = [] for wf, content in tempWfs.iteritems(): if content['RequestStatus'] in statusList: finalWfs.append(wf) print "Found %d wfs in not in active state" % len(finalWfs) else: finalWfs = [wfName] tempWfs = wfDBReader.getRequestByNames(wfName, True) print "Checking %s with status '%s'." % ( wfName, tempWfs[wfName]['RequestStatus']) wqDocs, wqInboxDocs = [], [] localWQDocs, localWQInboxDocs = [], [] for counter, wf in enumerate(finalWfs): if counter % 100 == 0: print "%d wfs queried ..." % counter # check whether there are workqueue docs wqDocIDs = wqBackend.getElements(WorkflowName=wf) if wqDocIDs: print "Found %d workqueue docs for %s, status %s" % ( len(wqDocIDs), wf, tempWfs[wf]['RequestStatus']) print wqDocIDs wqDocs.append(wqDocIDs) # check whether there are workqueue_inbox docs if wqInboxDB.documentExists(wf): print "Found workqueue_inbox doc for %s, status %s" % ( wf, tempWfs[wf]['RequestStatus']) # then retrieve the document wqInboxDoc = wqInboxDB.document(wf) wqInboxDocs.append(wqInboxDoc) # check local queue wqDocIDs = localWQBackend.getElements(WorkflowName=wf) if wqDocIDs: print "Found %d local workqueue docs for %s, status %s" % ( len(wqDocIDs), wf, tempWfs[wf]['RequestStatus']) print wqDocIDs localWQDocs.append(wqDocIDs) if localWQInboxDB.documentExists(wf): print "Found local workqueue_inbox doc for %s, status %s" % ( wf, tempWfs[wf]['RequestStatus']) wqInboxDoc = localWQInboxDB.document(wf) print wqInboxDoc localWQInboxDocs.append(wqInboxDoc) # TODO TODO TODO for the moment only deletes for a specific workflow if wfName: var = raw_input("\nCan we delete all these documents (Y/N)? ") if var == "Y": # deletes workqueue_inbox doc if wqInboxDoc: print "Deleting workqueue_inbox id %s and %s" % ( wqInboxDoc['_id'], wqInboxDoc['_rev']) wqInboxDB.delete_doc(wqInboxDoc['_id'], wqInboxDoc['_rev']) # deletes workqueue docs if wqDocIDs: print "Deleting workqueue docs %s" % wqDocIDs wqBackend.deleteElements( *[x for x in wqDocIDs if x['RequestName'] in wfName]) else: print "You are the boss, aborting it ...\n"
def main(): """ It will either delete docs in couchdb for the workflow you have provided or it will loop over the final (or almost final) states and ask for your permission to delete them. """ wfName = sys.argv[1] if len(sys.argv) == 2 else [] if 'WMAGENT_CONFIG' not in os.environ: os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py' config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"]) # Instantiating central services (couch stuff) # print "Central Couch URL : %s" % config.WorkloadSummary.couchurl # print "Central ReqMgr URL : %s\n" % config.AnalyticsDataCollector.centralRequestDBURL wfDBReader = RequestDBReader(config.AnalyticsDataCollector.centralRequestDBURL, couchapp = config.AnalyticsDataCollector.RequestCouchApp) # Central services wqBackend = WorkQueueBackend(config.WorkloadSummary.couchurl) wqInboxDB = Database('workqueue_inbox', config.WorkloadSummary.couchurl) # Local services localWQBackend = WorkQueueBackend(config.WorkQueueManager.couchurl, db_name = "workqueue_inbox") localWQInboxDB = Database('workqueue', config.WorkQueueManager.couchurl) statusList = ["failed", "epic-FAILED", "completed", "closed-out", "announced", "aborted", "aborted-completed", "rejected", "normal-archived", "aborted-archived", "rejected-archived"] for stat in final_status: # retrieve list of workflows in each status if not wfName: # options = {'include_docs': False} date_range = {'startkey': [2015,5,15,0,0,0], 'endkey': [2015,5,26,0,0,0]} # finalWfs = wfDBReader.getRequestByCouchView("bydate", options, date_range) tempWfs = wfDBReader.getRequestByCouchView("bydate", date_range) #print "Found %d wfs in status: %s" %(len(finalWfs), stat) finalWfs = [] for wf, content in tempWfs.iteritems(): if content['RequestStatus'] in statusList: finalWfs.append(wf) print "Found %d wfs in not in active state" % len(finalWfs) else: finalWfs = [wfName] tempWfs = wfDBReader.getRequestByNames(wfName, True) print "Checking %s with status '%s'." % (wfName, tempWfs[wfName]['RequestStatus']) wqDocs, wqInboxDocs = [], [] localWQDocs, localWQInboxDocs = [], [] for counter, wf in enumerate(finalWfs): if counter % 100 == 0: print "%d wfs queried ..." % counter # check whether there are workqueue docs wqDocIDs = wqBackend.getElements(WorkflowName = wf) if wqDocIDs: print "Found %d workqueue docs for %s, status %s" % (len(wqDocIDs), wf, tempWfs[wf]['RequestStatus']) print wqDocIDs wqDocs.append(wqDocIDs) # check whether there are workqueue_inbox docs if wqInboxDB.documentExists(wf): print "Found workqueue_inbox doc for %s, status %s" % (wf, tempWfs[wf]['RequestStatus']) # then retrieve the document wqInboxDoc = wqInboxDB.document(wf) wqInboxDocs.append(wqInboxDoc) # check local queue wqDocIDs = localWQBackend.getElements(WorkflowName = wf) if wqDocIDs: print "Found %d local workqueue docs for %s, status %s" % (len(wqDocIDs), wf, tempWfs[wf]['RequestStatus']) print wqDocIDs localWQDocs.append(wqDocIDs) if localWQInboxDB.documentExists(wf): print "Found local workqueue_inbox doc for %s, status %s" % (wf, tempWfs[wf]['RequestStatus']) wqInboxDoc = localWQInboxDB.document(wf) print wqInboxDoc localWQInboxDocs.append(wqInboxDoc) # TODO TODO TODO for the moment only deletes for a specific workflow if wfName: var = raw_input("\nCan we delete all these documents (Y/N)? ") if var == "Y": # deletes workqueue_inbox doc if wqInboxDoc: print "Deleting workqueue_inbox id %s and %s" % (wqInboxDoc['_id'], wqInboxDoc['_rev']) wqInboxDB.delete_doc(wqInboxDoc['_id'], wqInboxDoc['_rev']) # deletes workqueue docs if wqDocIDs: print "Deleting workqueue docs %s" % wqDocIDs wqBackend.deleteElements(*[x for x in wqDocIDs if x['RequestName'] in wfName]) else: print "You are the boss, aborting it ...\n"