def __init__(self, rest, config):
super(CouchDBCleanup, self).__init__(config)
self.reqDB = RequestDBReader(config.reqmgrdb_url)
# create another object to always retrieve up-to-date views (no stale)
self.reqDBNoStale = RequestDBReader(config.reqmgrdb_url)
self.reqDBNoStale._setNoStale()
self.reqmgrAux = ReqMgrAux(config.reqmgr2_url, logger=self.logger)
# statuses that we want to keep the transfer documents
self.transferStatuses = ["assigned", "staging", "staged", "acquired",
"failed", "running-open", "running-closed"]
baseURL, acdcDB = splitCouchServiceURL(config.acdc_url)
self.acdcService = CouchService(url=baseURL, database=acdcDB)
def __init__(self, app, api, config, mount):
RESTEntity.__init__(self, app, api, config, mount)
# CouchDB auxiliary database name
self.reqmgr_aux_db = api.db_handler.get_db(config.couch_reqmgr_aux_db)
self.reqmgr_aux_db_service = RequestDBReader(self.reqmgr_aux_db,
couchapp="ReqMgrAux")
self.setName()
def __init__(self, couchURL, appName="WMStats", reqdbURL=None, reqdbCouchApp="ReqMgr"):
self._sanitizeURL(couchURL)
# set the connection for local couchDB call
self._commonInit(couchURL, appName)
if reqdbURL:
self.reqDB = RequestDBReader(reqdbURL, reqdbCouchApp)
else:
self.reqDB = None
def __init__(self, couchURL, reqdbURL=None, reqdbCouchApp="ReqMgr"):
couchURL = sanitizeURL(couchURL)['url']
# set the connection for local couchDB call
self._commonInit(couchURL)
if reqdbURL:
self.reqDB = RequestDBReader(reqdbURL)
else:
self.reqDB = None
def getRequestInformationAndWorkload(requestName, reqmgrUrl, centralRequestDBURL):
"""
_getRequestInformationAndWorkload_
Retrieve the request information for assignment
and the full pickled workload.
"""
wfDBReader = RequestDBReader(centralRequestDBURL, couchapp="ReqMgr")
result = wfDBReader.getRequestByNames(requestName, True)
workloadDB = Database(result[requestName]['CouchWorkloadDBName'], result[requestName]['CouchURL'])
workloadPickle = workloadDB.getAttachment(requestName, 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
return workload, result[requestName]
def __init__(self, rest, config):
super(BuildParentLock, self).__init__(config)
self.reqmgrAux = ReqMgrAux(config.reqmgr2_url, logger=self.logger)
self.dbs = DBS3Reader(config.dbs_url)
# cache of dbs lookups mapping input dataset to parent dataset
self.dbsLookupCache = {}
# set of of currently active datasets requiring parent dataset
self.inputDatasetCache = set()
self.reqDB = RequestDBReader(config.reqmgrdb_url)
self.filterKeys = [
'assignment-approved', 'assigned', 'staging', 'staged', 'failed',
'acquired', 'running-open', 'running-closed', 'force-complete',
'completed', 'closed-out'
]
def setup(self, parameters):
"""
Called at startup
"""
# set the connection for local couchDB call
self.useReqMgrForCompletionCheck = getattr(
self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
self.archiveDelayHours = getattr(self.config.TaskArchiver,
'archiveDelayHours', 0)
self.wmstatsCouchDB = WMStatsWriter(
self.config.TaskArchiver.localWMStatsURL, "WMStatsAgent")
#TODO: we might need to use local db for Tier0
self.centralRequestDBReader = RequestDBReader(
self.config.AnalyticsDataCollector.centralRequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
if self.useReqMgrForCompletionCheck:
self.deletableState = "announced"
self.centralRequestDBWriter = RequestDBWriter(
self.config.AnalyticsDataCollector.centralRequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
if self.config.TaskArchiver.reqmgr2Only:
self.reqmgr2Svc = ReqMgr(
self.config.TaskArchiver.ReqMgr2ServiceURL)
else:
#TODO: remove this for reqmgr2
self.reqmgrSvc = RequestManager(
{'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
else:
# Tier0 case
self.deletableState = "completed"
# use local for update
self.centralRequestDBWriter = RequestDBWriter(
self.config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
jobDBName = self.config.JobStateMachine.couchDBName
self.jobCouchdb = CouchServer(jobDBurl)
self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" %
jobDBName)
self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" %
jobDBName)
statSummaryDBName = self.config.JobStateMachine.summaryStatsDBName
self.statsumdatabase = self.jobCouchdb.connectDatabase(
statSummaryDBName)
def gatherActiveDataStats(self, config):
"""
gather active data statistics
"""
try:
if DataCache.islatestJobDataExpired():
reqDB = RequestDBReader(config.requestDBURL)
wmstatsDB = WMStatsReader(config.wmstatsURL)
requestNames = reqDB.getRequestByStatus(ACTIVE_STATUS)
jobData = wmstatsDB.getLatestJobInfoByRequests(requestNames)
DataCache.setlatestJobData(jobData)
except Exception as ex:
self.logger.error(str(ex))
return
def fetchWorkflowsSpec(config, listOfWfs):
"""
Fetch the workload of a list of workflows. Filter out only a few
usefull keys
"""
if isinstance(listOfWfs, basestring):
listOfWfs = [listOfWfs]
wfDBReader = RequestDBReader(config.AnalyticsDataCollector.centralRequestDBURL,
couchapp=config.AnalyticsDataCollector.RequestCouchApp)
tempWfs = wfDBReader.getRequestByNames(listOfWfs, True)
wfShortDict = {}
for wf in listOfWfs:
wfShortDict[wf] = filterKeys(tempWfs[wf])
return wfShortDict
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["ReqMgr"]
self.testInit = TestInitCouchApp('RequestDBServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
dbName = 'requsetdb_t'
self.testInit.setupCouch(dbName, *self.couchApps)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
self.requestWriter = RequestDBWriter(reqDBURL)
self.requestReader = RequestDBReader(reqDBURL)
self.requestWriter.defaultStale = {}
self.requestReader.defaultStale = {}
return
def acdcCleanup(self, config):
"""
gather active data statistics
"""
reqDB = RequestDBReader(config.reqmgrdb_url)
from WMCore.ACDC.CouchService import CouchService
baseURL, acdcDB = splitCouchServiceURL(config.acdc_url)
acdcService = CouchService(url=baseURL, database=acdcDB)
originalRequests = acdcService.listCollectionNames()
if len(originalRequests) == 0:
return
# filter requests
results = reqDB._getCouchView("byrequest", {}, originalRequests)
# checkt he status of the requests [announced, rejected-archived, aborted-archived, normal-archived]
deleteStates = [
"announced", "rejected-archived", "aborted-archived",
"normal-archived"
]
filteredRequests = []
for row in results["rows"]:
if row["value"][0] in deleteStates:
filteredRequests.append(row["key"])
total = 0
for req in filteredRequests:
try:
deleted = acdcService.removeFilesetsByCollectionName(req)
if deleted == None:
self.logger.warning("request alread deleted %s", req)
else:
total += len(deleted)
self.logger.info("request %s deleted", req)
except Exception as ex:
self.logger.error(
"request deleted failed: will try again %s: %s", req,
str(ex))
self.logger.info("total %s requests deleted", total)
return
def main():
"""
It will either delete docs in couchdb for the workflow you
have provided or it will loop over the final (or almost final)
states and ask for your permission to delete them.
"""
args = sys.argv[1:]
if not len(args) == 1:
print "usage: python syncPrioReqMgrxGQ.py <text_file_with_the_workflow_names>"
sys.exit(0)
inputFile = args[0]
with open(inputFile) as f:
listWorkflows = [x.rstrip('\n') for x in f.readlines()]
if 'WMAGENT_CONFIG' not in os.environ:
os.environ[
'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
wfDBReader = RequestDBReader(
config.AnalyticsDataCollector.centralRequestDBURL,
couchapp=config.AnalyticsDataCollector.RequestCouchApp)
wqBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
workflowsDict = wfDBReader.getRequestByNames(listWorkflows)
for wf, details in workflowsDict.iteritems():
print "wf: %s and prio: %s" % (wf, details['RequestPriority'])
wqDocs = wqBackend.getElements(WorkflowName=wf)
docIds = [
elem._id for elem in wqDocs if elem['Status'] == 'Available'
and elem['Priority'] != details['RequestPriority']
]
if docIds:
print "Changing the priority of the following available docs: %s" % docIds
wqBackend.updateElements(*docIds,
Priority=details['RequestPriority'])
else:
print " there is nothing to update for this workflow."
def main():
"""
It will either delete docs in couchdb for the workflow you
have provided or it will loop over the final (or almost final)
states and ask for your permission to delete them.
"""
wfName = sys.argv[1] if len(sys.argv) == 2 else []
if 'WMAGENT_CONFIG' not in os.environ:
os.environ[
'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
# Instantiating central services (couch stuff)
# print "Central Couch URL : %s" % config.WorkloadSummary.couchurl
# print "Central ReqMgr URL : %s\n" % config.AnalyticsDataCollector.centralRequestDBURL
wfDBReader = RequestDBReader(
config.AnalyticsDataCollector.centralRequestDBURL,
couchapp=config.AnalyticsDataCollector.RequestCouchApp)
# Central services
wqBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
wqInboxDB = Database('workqueue_inbox', config.WorkloadSummary.couchurl)
# Local services
localWQBackend = WorkQueueBackend(config.WorkQueueManager.couchurl,
db_name="workqueue_inbox")
localWQInboxDB = Database('workqueue', config.WorkQueueManager.couchurl)
statusList = [
"failed", "epic-FAILED", "completed", "closed-out", "announced",
"aborted", "aborted-completed", "rejected", "normal-archived",
"aborted-archived", "rejected-archived"
]
for stat in final_status:
# retrieve list of workflows in each status
if not wfName:
# options = {'include_docs': False}
date_range = {
'startkey': [2015, 5, 15, 0, 0, 0],
'endkey': [2015, 5, 26, 0, 0, 0]
}
# finalWfs = wfDBReader.getRequestByCouchView("bydate", options, date_range)
tempWfs = wfDBReader.getRequestByCouchView("bydate", date_range)
#print "Found %d wfs in status: %s" %(len(finalWfs), stat)
finalWfs = []
for wf, content in tempWfs.iteritems():
if content['RequestStatus'] in statusList:
finalWfs.append(wf)
print "Found %d wfs in not in active state" % len(finalWfs)
else:
finalWfs = [wfName]
tempWfs = wfDBReader.getRequestByNames(wfName, True)
print "Checking %s with status '%s'." % (
wfName, tempWfs[wfName]['RequestStatus'])
wqDocs, wqInboxDocs = [], []
localWQDocs, localWQInboxDocs = [], []
for counter, wf in enumerate(finalWfs):
if counter % 100 == 0:
print "%d wfs queried ..." % counter
# check whether there are workqueue docs
wqDocIDs = wqBackend.getElements(WorkflowName=wf)
if wqDocIDs:
print "Found %d workqueue docs for %s, status %s" % (
len(wqDocIDs), wf, tempWfs[wf]['RequestStatus'])
print wqDocIDs
wqDocs.append(wqDocIDs)
# check whether there are workqueue_inbox docs
if wqInboxDB.documentExists(wf):
print "Found workqueue_inbox doc for %s, status %s" % (
wf, tempWfs[wf]['RequestStatus'])
# then retrieve the document
wqInboxDoc = wqInboxDB.document(wf)
wqInboxDocs.append(wqInboxDoc)
# check local queue
wqDocIDs = localWQBackend.getElements(WorkflowName=wf)
if wqDocIDs:
print "Found %d local workqueue docs for %s, status %s" % (
len(wqDocIDs), wf, tempWfs[wf]['RequestStatus'])
print wqDocIDs
localWQDocs.append(wqDocIDs)
if localWQInboxDB.documentExists(wf):
print "Found local workqueue_inbox doc for %s, status %s" % (
wf, tempWfs[wf]['RequestStatus'])
wqInboxDoc = localWQInboxDB.document(wf)
print wqInboxDoc
localWQInboxDocs.append(wqInboxDoc)
# TODO TODO TODO for the moment only deletes for a specific workflow
if wfName:
var = raw_input("\nCan we delete all these documents (Y/N)? ")
if var == "Y":
# deletes workqueue_inbox doc
if wqInboxDoc:
print "Deleting workqueue_inbox id %s and %s" % (
wqInboxDoc['_id'], wqInboxDoc['_rev'])
wqInboxDB.delete_doc(wqInboxDoc['_id'], wqInboxDoc['_rev'])
# deletes workqueue docs
if wqDocIDs:
print "Deleting workqueue docs %s" % wqDocIDs
wqBackend.deleteElements(
*[x for x in wqDocIDs if x['RequestName'] in wfName])
else:
print "You are the boss, aborting it ...\n"
