def main():
print "Looking for problematic inbox elements..."
problemRequests = getProblematicRequests()
print "Found %d bad elements:" % len(problemRequests)
if not problemRequests:
print "Nothing to fix, contact a developer if the problem persists..."
return 0
for request in problemRequests:
print request["RequestName"]
var = raw_input("Can we close these for new data in inbox elements: Y/N\n")
if var == "Y":
print "Updating them in global inbox, you need a WMAgent proxy for this."
inboxDB = Database('workqueue_inbox', 'https://cmsweb.cern.ch/couchdb')
for request in problemRequests:
inboxDB.document(request._id)
inboxDB.updateDocument(request._id,
'WorkQueue',
'in-place',
fields={'OpenForNewData': false})
print "Done with the deletions, this should fix the problem."
return 0
else:
var = raw_input("Then can we delete these inbox elements: Y/N\n")
if var == "Y":
print "Deleting them from the global inbox, you need a WMAgent proxy for this."
inboxDB = Database('workqueue_inbox',
'https://cmsweb.cern.ch/couchdb')
for request in problemRequests:
inboxDB.delete_doc(request._id, request.rev)
print "Done with the deletions, this should fix the problem."
return 0
else:
print "Doing nothing as you commanded..."
return 0
def testSlashInDBName(self):
"""
Slashes are a valid character in a database name, and are useful as it
creates a directory strucutre for the couch data files.
"""
db_name = 'wmcore/unittests'
try:
self.server.deleteDatabase(db_name)
except:
# Ignore this - the database shouldn't already exist
pass
db = self.server.createDatabase(db_name)
info = db.info()
assert info['db_name'] == db_name
db_name = 'wmcore/unittests'
db = self.server.connectDatabase(db_name)
info = db.info()
assert info['db_name'] == db_name
db = Database(db_name, url=os.environ["COUCHURL"])
info = db.info()
assert info['db_name'] == db_name
self.server.deleteDatabase(db_name)
def changePriority(requestName, priority, wmstatUrl = None):
"""
Changes the priority that's stored in the workload.
Takes the current priority stored in the workload and adds
to it the input priority value.
"""
request = requestDetails(requestName)
# change in Oracle
newPrior = int(priority)
ChangeState.changeRequestPriority(requestName, newPrior)
# change in workload (spec)
helper = loadWorkload(request)
helper.data.request.priority = newPrior
saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
# change priority in CouchDB
couchDb = Database(request["CouchWorkloadDBName"], request["CouchURL"])
fields = {"RequestPriority": newPrior}
couchDb.updateDocument(requestName, "ReqMgr", "updaterequest", fields=fields)
# push the change to the WorkQueue
response = ProdManagement.getProdMgr(requestName)
if response == [] or response[0] is None or response[0] == "":
# Request must not be assigned yet, we are safe here
return
workqueue = WorkQueue.WorkQueue(response[0])
workqueue.updatePriority(requestName, priority)
return
def changePriority(requestName, priority, wmstatUrl=None):
"""
Changes the priority that's stored in the workload.
Takes the current priority stored in the workload and adds
to it the input priority value.
"""
request = requestDetails(requestName)
# change in Oracle
newPrior = int(priority)
ChangeState.changeRequestPriority(requestName, newPrior)
# change in workload (spec)
helper = loadWorkload(request)
helper.data.request.priority = newPrior
saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
# change priority in CouchDB
couchDb = Database(request["CouchWorkloadDBName"], request["CouchURL"])
fields = {"RequestPriority": newPrior}
couchDb.updateDocument(requestName,
"ReqMgr",
"updaterequest",
fields=fields,
useBody=True)
# push the change to the WorkQueue
gqURL = "%s/workqueue" % request["CouchURL"]
workqueue = WorkQueue.WorkQueue(gqURL)
workqueue.updatePriority(requestName, priority)
return
def __init__(self, config):
# configuration values:
# 'uri' attribute (URL of the REST server and resource name)
# in case of CouchDB, the resource name is the database name
# http://servername:port/databaseName
self.config = config
logging.info("Instantiating ...")
# the class currently relies only on 1 REST server possibility - the
# CouchDB server. as explained above, .database will be replaced by
# .connection if both a generic REST server as well as CouchDB are to
# be talked to
split = self.config.uri.rfind('/')
dbName = self.config.uri[split +
1:] # get last item of URI - database name
url = self.config.uri[:split]
# as opposed to CouchSink, here it's assumed the resource (the database name)
# does exist, fail here otherwise
# this check / rest of the constructed may be revised for
# general REST server
server = CouchServer(url)
databases = server.listDatabases()
# there needs to be this database created upfront and also
# couchapp associated with it installed, if it's there, fail
if dbName not in databases:
raise Exception("REST URI: %s (DB name: %s) does not exist." %
(self.config.uri, dbName))
self._database = Database(dbName, url)
logging.info("Initialized.")
def main():
"""
_main_
"""
usage = "Usage: python %prog -w workflow"
parser = OptionParser(usage=usage)
parser.add_option('-w',
'--workflow',
help='Workflow name in ReqMgr',
dest='wf')
(options, args) = parser.parse_args()
if not options.wf:
parser.error('You must provide a workflow name')
sys.exit(1)
couchUrl = "https://cmsweb.cern.ch/couchdb"
database = "acdcserver"
failures = {}
svc = Database(database, couchUrl)
result = svc.loadView("ACDC", "byCollectionName", {
'key': options.wf,
'include_docs': True,
'reduce': False
})
print "Found %i failures/rows in total." % len(result["rows"])
for entry in result["rows"]:
if entry['doc']['fileset_name'] in failures:
failures[entry['doc']['fileset_name']] += 1
else:
failures[entry['doc']['fileset_name']] = 1
pprint(failures)
print "\nDone!"
def testA(self):
""" make some documents and own them"""
guInt = Interface(self.testInit.couchUrl, self.testInit.couchDbName)
#create a couple of docs
couch = Database(self.testInit.couchDbName, self.testInit.couchUrl)
for x in range(10):
doc = Document("document%s" % x, {"Test Data": [1,2,3,4] })
couch.queue(doc)
couch.commit()
self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 0)
self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 0)
guInt.callUpdate("ownthis","document1", group = self.owner1.group.name, user = self.owner1.name)
self.assertTrue("document1" in guInt.documentsOwned(self.owner1.group.name, self.owner1.name))
self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 1)
self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 0)
guInt.callUpdate("ownthis","document2", group = self.owner2.group.name, user = self.owner2.name)
self.assertTrue("document2" in guInt.documentsOwned(self.owner2.group.name, self.owner2.name))
self.assertEqual(len(guInt.documentsOwned(self.owner1.group.name, self.owner1.name)), 1)
self.assertEqual(len(guInt.documentsOwned(self.owner2.group.name, self.owner2.name)), 1)
guInt.callUpdate("newgroup", "group-DataOps", group = "DataOps")
self.assertTrue(couch.documentExists("group-DataOps") )
guInt.callUpdate("newuser", "user-damason", group = "DataOps", user = "******")
self.assertTrue(couch.documentExists("user-damason") )
def checkWorkQueue(requestName):
result = {'ActiveAgents' : {},
'ElementsRunning' : 0,
'ElementsAcquired' : 0,
'ElementsAvailable' : 0,
'ElementsDone' : 0}
x = Database('workqueue', 'https://cmsweb.cern.ch/couchdb')
y = x.loadView('WorkQueue', 'elementsByParent', {'include_docs' : True}, [requestName])
for entry in y['rows']:
doc = entry['doc']
element = doc['WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement']
status = element['Status']
if status == 'Running':
result['ElementsRunning'] += 1
elif status == 'Acquired':
result['ElementsAcquired'] += 1
elif status == 'Available':
result['ElementsAvailable'] += 1
elif status == 'Done':
result['ElementsDone'] += 1
if status not in ['Done', 'Available']:
agent = element['ChildQueueUrl']
if agent not in result['ActiveAgents']:
result['ActiveAgents'][agent] = 0
result['ActiveAgents'][agent] += 1
return result
def setUp(self):
self.testInit = TestInitCouchApp("ReproducibleSeedingTest")
self.testInit.setupCouch("seeding_config_cache", "GroupUser",
"ConfigCache")
self.database = Database(self.testInit.couchDbName,
self.testInit.couchUrl)
self.documentId = None
def main():
wmstatsDB = Database('wmstats', 'https://alancc7-cloud3.cern.ch/couchdb')
conflictDocs = wmstatsDB.loadView("WMStats3", "conflicts")
print("Found {} conflicting documents".format(
conflictDocs.get("total_rows")))
print(" they are:\n{}".format(pformat(conflictDocs.get("rows", []))))
for doc in conflictDocs.get("rows", []):
resolveConflict(doc)
def getSplitting(requestName):
reqmgrUrl = 'https://cmsweb.cern.ch/reqmgr/reqMgr/'
reqmgr = RequestManager(dict={'endpoint': reqmgrUrl})
result = reqmgr.getRequest(requestName)
workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
workloadPickle = workloadDB.getAttachment(requestName, 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
params = workload.getTopLevelTask()[0].jobSplittingParameters()
algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm()
return params['events_per_job']
def setUp(self):
"""
_setUp_
Setup a couch database for testing
of produced JSON
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("histogram_dump_t")
self.histogramDB = Database(dbname="histogram_dump_t")
def _getDbConnection(couchUrl, dbName):
"""
Check if the database exists, create if not.
"""
couchServer = CouchServer(couchUrl)
if not dbName in couchServer.listDatabases():
logging.info("Database '%s' does not exits, creating it." % dbName)
db = couchServer.createDatabase(dbName) # returns Database
else:
logging.debug("Database '%s' exists." % dbName)
db = Database(dbName, couchUrl)
return couchServer, db
def getRequestInformationAndWorkload(requestName, reqmgrUrl, centralRequestDBURL):
"""
_getRequestInformationAndWorkload_
Retrieve the request information for assignment
and the full pickled workload.
"""
wfDBReader = RequestDBReader(centralRequestDBURL, couchapp="ReqMgr")
result = wfDBReader.getRequestByNames(requestName, True)
workloadDB = Database(result[requestName]['CouchWorkloadDBName'], result[requestName]['CouchURL'])
workloadPickle = workloadDB.getAttachment(requestName, 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
return workload, result[requestName]
def getRequestInformationAndWorkload(requestName, reqmgrUrl):
"""
_getRequestInformationAndWorkload_
Retrieve the request information for assignment
and the full pickled workload.
"""
reqmgr = RequestManager(dict={'endpoint': reqmgrUrl})
result = reqmgr.getRequest(requestName)
workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL'])
workloadPickle = workloadDB.getAttachment(requestName, 'spec')
spec = pickle.loads(workloadPickle)
workload = WMWorkloadHelper(spec)
return workload, result
def retrieveResubmissionChildren(requestName, couchUrl, couchDBName):
"""
_retrieveResubmissionChildren_
Construct a list of request names which are the resubmission
offspring from a request. This is a recursive
call with a single requestName as input.
The result only includes the children and not the original request.
"""
childrenRequestNames = []
reqmgrDb = Database(couchDBName, couchUrl)
result = reqmgrDb.loadView('ReqMgr', 'childresubmissionrequests', keys = [requestName])['rows']
for child in result:
childrenRequestNames.append(child['id'])
childrenRequestNames.extend(retrieveResubmissionChildren(child['id'], couchUrl, couchDBName))
return childrenRequestNames
def __init__(self, config):
self.config = config
logging.info("Instantiating ...")
# test if the configured database does not exist, create it
server = CouchServer(self.config.url)
databases = server.listDatabases()
if self.config.database not in databases:
logging.warn(
"'%s' database does not exist on %s, creating it ..." %
(self.config.database, self.config.url))
server.createDatabase(self.config.database)
logging.warn("Created.")
logging.info("'%s' database exists on %s" %
(self.config.database, self.config.url))
self.database = Database(self.config.database, self.config.url)
logging.info("Initialized.")
def main():
if len(sys.argv) < 2:
print("Takes 1 input argument - dump of Oracle reqmgr_request "
"table in a Python dictionary.")
sys.exit(1)
print("Creating database connection ...")
# couch_server = CouchServer(couch_url)
db = Database(couch_db_name, couch_url)
execfile(sys.argv[1], globals())
oracle_requests = reqmgr_request # read from the input file
print("Oracle requests: %s" % len(oracle_requests))
print("Retrieving data from CouchDB ...")
couch_requests = db.allDocs()
couch_request_names = []
for row in couch_requests["rows"]:
if row["id"].startswith("_design"): continue
couch_request_names.append(row["id"])
print("CouchDB requests: %s" % len(couch_request_names))
print("Comparing Oracle and CouchDB requests ...")
not_present_in_couch = []
for request in oracle_requests:
oracle_request_name = request["REQUEST_NAME"]
# remove first occurrence of value. Raises ValueError if not present
try:
couch_request_names.remove(oracle_request_name)
except ValueError:
not_present_in_couch.append(oracle_request_name)
print("CouchDB requests not present in Oracle:")
print("%s requests" % len(couch_request_names))
for name in couch_request_names:
request = db.document(name)
if name != request["RequestName"] or name != request["_id"]:
print(
"\t Mismatch: CouchDB id: '%s' RequestName: '%s' name: '%s'" %
(request["_id"], request["RequestName"], name))
print("%s %s %s" % (request["RequestName"], request["RequestType"],
request["RequestStatus"]))
print("\n\n")
print("Oracle requests not present in CouchDB:")
print("%s requests" % len(not_present_in_couch))
for name in not_present_in_couch:
print(name)
def main():
if len(sys.argv) != 2:
print "Usage:"
print "python CheckWorkQueueElements.py <workflowName>"
sys.exit(0)
workflow = sys.argv[1]
x = Database('workqueue', 'https://cmsweb.cern.ch/couchdb')
y = x.loadView('WorkQueue', 'elementsByParent', {'include_docs': True},
[workflow])
for entry in y['rows']:
doc = entry['doc']
element = doc[
'WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement']
if element['Status'] != 'Done':
print 'Element: %s is %s in %s' % (doc['_id'], element['Status'],
element['ChildQueueUrl'])
def changeRequestStatus(requestName, newState, priority=None, wmstatUrl=None):
"""
_changeRequestStatus_
Basic API to change a request to a new state, also
includes optional priority change for the request
- *requestName* : name of the request to be modified
- *newState* : name of the new status for the request
- *priority* : optional integer priority
Apparently when changing request state (on assignment page),
it's possible to change priority at one go. Hence the argument is here.
"""
# MySQL/Oracle
factory = DBConnect.getConnection()
reqId = getRequestID(factory, requestName)
changeRequestIDStatus(reqId, newState, priority)
# CouchDB
# have to first get information where the request Couch document is,
# extracting the information from reqmgr_request.workflow table field
reqData = factory(classname="Request.Get").execute(reqId)
# this would be something like this:
# http://localhost:5984/reqmgr_workload_cache/maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
wfUrl = reqData['workflow']
# cut off /maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
couchUrl = wfUrl.replace('/' + requestName + "/spec", '')
couchDbName = couchUrl[couchUrl.rfind('/') + 1:]
# cut off database name from the URL
url = couchUrl.replace('/' + couchDbName, '')
couchDb = Database(couchDbName, url)
fields = {"RequestStatus": newState}
couchDb.updateDocument(requestName,
"ReqMgr",
"updaterequest",
fields=fields,
useBody=True)
#TODO: should we make this mendatory?
if wmstatUrl:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.updateRequestStatus(requestName, newState)
def _getDbConnection(couchUrl, dbName):
"""
Check if the database exists, create if not.
"""
couchServer = CouchServer(couchUrl)
if not dbName in couchServer.listDatabases():
logging.info("Database '%s' does not exits, creating it." % dbName)
db = couchServer.createDatabase(dbName)
else:
logging.debug("Database '%s' exists." % dbName)
db = Database(dbName, couchUrl)
couchapps = "../../../src/couchapp"
stat_couchapp = "%s/stat" % couchapps
harness = CouchAppTestHarness(dbName, couchUrl)
harness.create()
harness.pushCouchapps(stat_couchapp)
return couchServer, db
def update_software(config_file):
"""
Functions retrieves CMSSW versions and scramarchs from CMS tag collector.
"""
config = loadConfigurationFile(config_file)
# source of the data
tag_collector_url = config.views.data.tag_collector_url
# store the data into CouchDB auxiliary database under "software" document
couch_host = config.views.data.couch_host
reqmgr_aux_db = config.views.data.couch_reqmgr_aux_db
# get data from tag collector
all_archs_and_versions = _get_all_scramarchs_and_versions(
tag_collector_url)
if not all_archs_and_versions:
return
# get data already stored in CouchDB
couchdb = Database(dbname=reqmgr_aux_db, url=couch_host)
try:
sw_already_stored = couchdb.document("software")
del sw_already_stored["_id"]
del sw_already_stored["_rev"]
except CouchNotFoundError:
logging.error("Document id software, does not exist, creating it ...")
doc = Document(id="software", inputDict=all_archs_and_versions)
couchdb.commitOne(doc)
return
# now compare recent data from tag collector and what we already have stored
# sorting is necessary
if sorted(all_archs_and_versions) != sorted(sw_already_stored):
logging.debug(
"ScramArch/CMSSW releases changed, updating software document ...")
doc = Document(id="software", inputDict=all_archs_and_versions)
couchdb.commitOne(doc)
"""
def test06UploadACDC(self):
# get previous request we can piggyback on
for request in reversed(self.__class__.reqmgr.getRequest()):
request = request[
'WMCore.RequestManager.DataStructs.Request.Request'][
'RequestName']
if 'RequestCancellation_t' in request:
self.__class__.requestParams['OriginalRequestName'] = request
break
else:
raise nose.SkipTest("no suitable request in reqmgr to resubmit")
self.__class__.requestParams[
'InitialTaskPath'] = self.__class__.requestParams[
'InitialTaskPath'] % self.__class__.requestParams[
'OriginalRequestName']
self.__class__.requestParams[
'ACDCServer'] = self.__class__.endpoint + '/couchdb'
# create and upload acdc
service = DataCollectionService(url=self.__class__.endpoint +
'/couchdb',
database='wmagent_acdc')
service.createCollection(
self.__class__.requestParams['OriginalRequestName'], 'integration',
'DMWM')
with open(
os.path.join(
getTestBase(), '..', 'data', 'ACDC',
'linacre_ACDC_ReReco13JulCosmics_120809_130020_117_120823_200309_5735.json'
)) as infile:
acdc_json = infile.read().replace(
'linacre_ACDC_ReReco13JulCosmics_120809_130020_117_120823_200309_5735',
self.__class__.requestParams['OriginalRequestName'])
acdc_json = loads(acdc_json)
acdc_database = Database('wmagent_acdc',
self.__class__.endpoint + '/couchdb')
acdc_database.commit(acdc_json)
def dump(full_dump=False, fields=None):
print "Querying fields: %s\n\n" % fields
db = Database(couch_db_name, couch_url)
couch_requests = db.allDocs()
doc_counter = 0
for row in couch_requests["rows"]:
if row["id"].startswith("_design"): continue
doc = db.document(row["id"])
if fields:
s = ''
for f in fields:
try:
s += "%s:%s " % (f, doc[f])
except KeyError:
s += "%s:n/a " % f
print "%s %s\n" % (s, doc["RequestName"])
elif full_dump:
print "%s\n%s\n%s\n" % (row["id"], doc, 70 * '-')
else:
print row["id"]
doc_counter += 1
#if doc_counter > 100:
# break
print "Total documents: %s" % doc_counter
kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(
kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'],
self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"],
"ReqMgr",
"updaterequest",
fields={
"AcquisitionEra":
reqDetails["AcquisitionEra"],
"Teams":
JsonWrapper.JSONEncoder().encode(
kwargs["Teams"]),
"SiteWhitelist":
JsonWrapper.JSONEncoder().encode(whiteList),
"SiteBlacklist":
JsonWrapper.JSONEncoder().encode(blackList)
})
def main():
"""
It will either delete docs in couchdb for the workflow you
have provided or it will loop over the final (or almost final)
states and ask for your permission to delete them.
"""
wfName = sys.argv[1] if len(sys.argv) == 2 else []
if 'WMAGENT_CONFIG' not in os.environ:
os.environ[
'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
# Instantiating central services (couch stuff)
# print "Central Couch URL : %s" % config.WorkloadSummary.couchurl
# print "Central ReqMgr URL : %s\n" % config.AnalyticsDataCollector.centralRequestDBURL
wfDBReader = RequestDBReader(
config.AnalyticsDataCollector.centralRequestDBURL,
couchapp=config.AnalyticsDataCollector.RequestCouchApp)
# Central services
wqBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
wqInboxDB = Database('workqueue_inbox', config.WorkloadSummary.couchurl)
# Local services
localWQBackend = WorkQueueBackend(config.WorkQueueManager.couchurl,
db_name="workqueue_inbox")
localWQInboxDB = Database('workqueue', config.WorkQueueManager.couchurl)
statusList = [
"failed", "epic-FAILED", "completed", "closed-out", "announced",
"aborted", "aborted-completed", "rejected", "normal-archived",
"aborted-archived", "rejected-archived"
]
for stat in final_status:
# retrieve list of workflows in each status
if not wfName:
# options = {'include_docs': False}
date_range = {
'startkey': [2015, 5, 15, 0, 0, 0],
'endkey': [2015, 5, 26, 0, 0, 0]
}
# finalWfs = wfDBReader.getRequestByCouchView("bydate", options, date_range)
tempWfs = wfDBReader.getRequestByCouchView("bydate", date_range)
#print "Found %d wfs in status: %s" %(len(finalWfs), stat)
finalWfs = []
for wf, content in tempWfs.iteritems():
if content['RequestStatus'] in statusList:
finalWfs.append(wf)
print "Found %d wfs in not in active state" % len(finalWfs)
else:
finalWfs = [wfName]
tempWfs = wfDBReader.getRequestByNames(wfName, True)
print "Checking %s with status '%s'." % (
wfName, tempWfs[wfName]['RequestStatus'])
wqDocs, wqInboxDocs = [], []
localWQDocs, localWQInboxDocs = [], []
for counter, wf in enumerate(finalWfs):
if counter % 100 == 0:
print "%d wfs queried ..." % counter
# check whether there are workqueue docs
wqDocIDs = wqBackend.getElements(WorkflowName=wf)
if wqDocIDs:
print "Found %d workqueue docs for %s, status %s" % (
len(wqDocIDs), wf, tempWfs[wf]['RequestStatus'])
print wqDocIDs
wqDocs.append(wqDocIDs)
# check whether there are workqueue_inbox docs
if wqInboxDB.documentExists(wf):
print "Found workqueue_inbox doc for %s, status %s" % (
wf, tempWfs[wf]['RequestStatus'])
# then retrieve the document
wqInboxDoc = wqInboxDB.document(wf)
wqInboxDocs.append(wqInboxDoc)
# check local queue
wqDocIDs = localWQBackend.getElements(WorkflowName=wf)
if wqDocIDs:
print "Found %d local workqueue docs for %s, status %s" % (
len(wqDocIDs), wf, tempWfs[wf]['RequestStatus'])
print wqDocIDs
localWQDocs.append(wqDocIDs)
if localWQInboxDB.documentExists(wf):
print "Found local workqueue_inbox doc for %s, status %s" % (
wf, tempWfs[wf]['RequestStatus'])
wqInboxDoc = localWQInboxDB.document(wf)
print wqInboxDoc
localWQInboxDocs.append(wqInboxDoc)
# TODO TODO TODO for the moment only deletes for a specific workflow
if wfName:
var = raw_input("\nCan we delete all these documents (Y/N)? ")
if var == "Y":
# deletes workqueue_inbox doc
if wqInboxDoc:
print "Deleting workqueue_inbox id %s and %s" % (
wqInboxDoc['_id'], wqInboxDoc['_rev'])
wqInboxDB.delete_doc(wqInboxDoc['_id'], wqInboxDoc['_rev'])
# deletes workqueue docs
if wqDocIDs:
print "Deleting workqueue docs %s" % wqDocIDs
wqBackend.deleteElements(
*[x for x in wqDocIDs if x['RequestName'] in wfName])
else:
print "You are the boss, aborting it ...\n"
def main():
"""
_main_
"""
usage = "Usage: python %prog -w workflow"
parser = OptionParser(usage=usage)
parser.add_option('-w',
'--workflow',
help='Workflow name in ReqMgr',
dest='wf')
parser.add_option('-v',
'--verbose',
help='Enable verbose mode',
action="store_true")
(options, args) = parser.parse_args()
if not options.wf:
parser.error('You must provide a workflow name')
sys.exit(1)
couchUrl = "https://cmsweb.cern.ch/couchdb"
database = "acdcserver"
failures = {}
svc = Database(database, couchUrl)
result = svc.loadView("ACDC", "byCollectionName", {
'key': options.wf,
'include_docs': True,
'reduce': False
})
print("Found %i failures/rows in total." % len(result["rows"]))
for entry in result["rows"]:
fsetName = entry['doc']['fileset_name']
failures.setdefault(
fsetName, {
'jobs': 0,
'files': 0,
'lumis': 0,
'uniqueLumis': 0,
'listLumis': set()
})
failures[fsetName]['jobs'] += 1
failures[fsetName]['files'] += len(entry['doc']['files'])
for fname in entry['doc']['files']:
for runLumi in entry['doc']['files'][fname]['runs']:
failures[fsetName]['lumis'] += len(runLumi['lumis'])
failures[fsetName]['listLumis'] |= set(runLumi['lumis'])
for fsetName in failures:
failures[fsetName]['uniqueLumis'] = len(
failures[fsetName]['listLumis'])
failures[fsetName]['listLumis'] = str(failures[fsetName]['listLumis'])
if not options.verbose:
for fset in failures.keys():
failures[fset].pop('listLumis', None)
print("Summary of failures is as follows:")
pprint(failures)
print("\nNow printing duplicate files + run + lumis per fileset")
printDups(result["rows"])
print("\nDone!")
def main():
if len(sys.argv) < 2:
print("Missing the connect Oracle TNS argument (user/password@server).")
sys.exit(1)
tns = sys.argv[1]
print("Creating CouchDB database connection ...")
couchdb = Database(couchdb_name, couch_url)
print("Creating Oracle database connection ...")
oradb = cx_Oracle.Connection(tns)
num_couch_requests = get_couchdb_row_count(couchdb)
print("Total CouchDB request documents in ReqMgr: %s" % num_couch_requests)
num_oracle_requests = get_oracle_row_count(oradb, "reqmgr_request")
print("Total Oracle requests entries in ReqMgr: %s" % num_oracle_requests)
if num_couch_requests != num_oracle_requests:
print("Number of requests in Oracle, CouchDB don't agree, fix that first.")
sys.exit(1)
else:
print("Database cross-check (Oracle request names vs CouchDB): DONE, THE SAME.")
def get_couch_value(couch_req, mapping):
try:
c = couch_req[mapping["couch"]]
couch_missing = False
except KeyError:
# comparison will not happen due to missing flag, the value
# will be stored in couch
c = "N/A"
couch_missing = False
return str(c), couch_missing
def check_oracle_worflow_value(oracle_value, mapping, req_name):
# check Oracle WORKFLOW value
if mapping["oracle"] == "WORKFLOW":
# https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/linacre_2011A_442p2_DataReprocessingMuOnia_111119_005717/spec
from_wf_url_req_name = oracle_value.rsplit('/', 2)[-2]
if req_name != from_wf_url_req_name:
print("Workflow URL mismatch: %s" % o)
sys.exit(1)
counter = 0
for oracle_req in get_oracle_data(oradb):
req_name = oracle_req["REQUEST_NAME"]
# FILTER
# check only requests injected approx. after last deployment (a lot of
# stuff should have already been fixed in ReqMgr)
# _13041._*$ (ending of request name with date/time)
#if not re.match(".*_1304[0-3][0-9]_.*$", req_name): # all April 2013
# continue
counter += 1
print("\n\n%s (%s)" % (req_name, counter))
couch_req = couchdb.document(req_name)
couch_fields_to_correct = {}
for mapping in MAPPING:
if mapping["couch"] in COUCH_TO_IGNORE:
continue
o = str(oracle_req[mapping["oracle"]])
c, couch_missing = get_couch_value(couch_req, mapping)
check_oracle_worflow_value(o, mapping, req_name)
# compare oracle and couch values
# don't update value in couch if it exists and is non-empty
if (couch_missing or o != c) and c in ('None', '0', '', "N/A"):
print("%s %s != %s" % (mapping, o, c))
# correct couch request by oracle value
couch_fields_to_correct[mapping["couch"]] = o
if couch_fields_to_correct:
print("Couch corrected fields:")
print(couch_fields_to_correct)
if sys.argv[-1] == "-c":
couchdb.updateDocument(req_name, "ReqMgr", "updaterequest",
fields=couch_fields_to_correct, useBody=True)
print("Couch updated")
else:
print("OK")
# fields that should be removed from couch
"""
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
#Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if type(whiteList) != list:
whiteList = [whiteList]
if type(blackList) != list:
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList,
siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(
400, "White and blacklist the same site is not allowed %s" %
list(res))
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(
400,
"Bad output dataset name, check the processed dataset.\n %s"
% str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
useAAA = strToBool(kwargs.get("useSiteListAsLocation", False))
if useAAA:
helper.setLocationDataSourceFlag(flag=useAAA)
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
for site in autoApproveList:
if site.endswith('_MSS'):
raise cherrypy.HTTPError(
400, "Auto-approval to MSS endpoint not allowed %s" %
autoApproveList)
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(
400, "Invalid subscription priority %s" % subscriptionPriority)
custodialType = kwargs.get("CustodialSubType", "Replica")
if custodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(
400, "Invalid custodial subscription type %s" % custodialType)
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
if nonCustodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(
400,
"Invalid noncustodial subscription type %s" % nonCustodialType)
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
priority=subscriptionPriority)
# Block closing information
blockCloseMaxWaitTime = int(
kwargs.get("BlockCloseMaxWaitTime",
helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(
kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(
kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(
kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
# set Task properties if they are exist
# TODO: need to define the task format (maybe kwargs["tasks"]?)
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'],
self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"],
"ReqMgr",
"updaterequest",
fields={
"AcquisitionEra":
reqDetails["AcquisitionEra"],
"ProcessingVersion":
reqDetails["ProcessingVersion"],
"CustodialSites":
custodialList,
"NonCustodialSites":
nonCustodialList,
"AutoApproveSubscriptionSites":
autoApproveList,
"SubscriptionPriority":
subscriptionPriority,
"CustodialSubType":
custodialType,
"NonCustodialSubType":
nonCustodialType,
"Teams":
kwargs["Teams"],
"OutputDatasets":
outputDatasets,
"SiteWhitelist":
whiteList,
"SiteBlacklist":
blackList
},
useBody=True)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
try:
helper.validateArgumentForAssignment(kwargs)
except WMSpecFactoryException as ex:
raise cherrypy.HTTPError(400, str(ex.message()))
except Exception:
msg = traceback.format_exc()
raise cherrypy.HTTPError(400, "Unhandled error: %s" % msg)
# Validate the different parts of the processed dataset
processedDatasetParts = {"AcquisitionEra": kwargs.get("AcquisitionEra"),
"ProcessingString": kwargs.get("ProcessingString"),
"ProcessingVersion": kwargs.get("ProcessingVersion")
}
for field, values in processedDatasetParts.iteritems():
if field in kwargs and isinstance(kwargs[field], dict):
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs.get(field, values), field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if not isinstance(whiteList, list):
whiteList = [whiteList]
if not isinstance(blackList, list):
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
helper.setAcquisitionEra(kwargs.get("AcquisitionEra", None))
helper.setProcessingString(kwargs.get("ProcessingString", None))
helper.setProcessingVersion(kwargs.get("ProcessingVersion", None))
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(400,
"Bad output dataset name, check the processed dataset.\n %s" %
str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
# FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
helper.setTrustLocationFlag(inputFlag=strToBool(kwargs.get("TrustSitelists", False)),
pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False)))
helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False)))
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
custodialType = kwargs.get("CustodialSubType", "Replica")
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
helper.setSubscriptionInformationWildCards(wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
custodialGroup=kwargs.get("CustodialGroup", "DataOps"),
nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"),
priority=subscriptionPriority,
deleteFromSource=kwargs.get("DeleteFromSource", False))
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setMemory(kwargs.get("Memory"))
helper.setCores(kwargs.get("Multicore"))
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"],
"ProcessingVersion": reqDetails["ProcessingVersion"],
"CustodialSites": custodialList,
"NonCustodialSites": nonCustodialList,
"AutoApproveSubscriptionSites": autoApproveList,
"SubscriptionPriority": subscriptionPriority,
"CustodialSubType": custodialType,
"NonCustodialSubType": nonCustodialType,
"CustodialGroup": kwargs.get("CustodialGroup", "DataOps"),
"NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"),
"DeleteFromSource": kwargs.get("DeleteFromSource", False),
"Teams": kwargs["Teams"],
"OutputDatasets": outputDatasets,
"SiteWhitelist": whiteList,
"SiteBlacklist": blackList,
"MergedLFNBase": kwargs["MergedLFNBase"],
"UnmergedLFNBase": kwargs["UnmergedLFNBase"],
"Dashboard": kwargs.get("Dashboard", ""),
"TrustSitelists": kwargs.get("TrustSitelists", False),
"TrustPUSitelists": kwargs.get("TrustPUSitelists", False),
"AllowOpportunistic": kwargs.get("AllowOpportunistic", False)},
useBody=True)
