def changePriority(requestName, priority, wmstatUrl=None):
"""
Changes the priority that's stored in the workload.
Takes the current priority stored in the workload and adds
to it the input priority value.
"""
request = requestDetails(requestName)
# change in Oracle
newPrior = int(priority)
ChangeState.changeRequestPriority(requestName, newPrior)
# change in workload (spec)
helper = loadWorkload(request)
helper.data.request.priority = newPrior
saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
# change priority in CouchDB
couchDb = Database(request["CouchWorkloadDBName"], request["CouchURL"])
fields = {"RequestPriority": newPrior}
couchDb.updateDocument(requestName,
"ReqMgr",
"updaterequest",
fields=fields,
useBody=True)
# push the change to the WorkQueue
gqURL = "%s/workqueue" % request["CouchURL"]
workqueue = WorkQueue.WorkQueue(gqURL)
workqueue.updatePriority(requestName, priority)
return
def main():
print "Looking for problematic inbox elements..."
problemRequests = getProblematicRequests()
print "Found %d bad elements:" % len(problemRequests)
if not problemRequests:
print "Nothing to fix, contact a developer if the problem persists..."
return 0
for request in problemRequests:
print request["RequestName"]
var = raw_input("Can we close these for new data in inbox elements: Y/N\n")
if var == "Y":
print "Updating them in global inbox, you need a WMAgent proxy for this."
inboxDB = Database('workqueue_inbox', 'https://cmsweb.cern.ch/couchdb')
for request in problemRequests:
inboxDB.document(request._id)
inboxDB.updateDocument(request._id,
'WorkQueue',
'in-place',
fields={'OpenForNewData': false})
print "Done with the deletions, this should fix the problem."
return 0
else:
var = raw_input("Then can we delete these inbox elements: Y/N\n")
if var == "Y":
print "Deleting them from the global inbox, you need a WMAgent proxy for this."
inboxDB = Database('workqueue_inbox',
'https://cmsweb.cern.ch/couchdb')
for request in problemRequests:
inboxDB.delete_doc(request._id, request.rev)
print "Done with the deletions, this should fix the problem."
return 0
else:
print "Doing nothing as you commanded..."
return 0
def changePriority(requestName, priority, wmstatUrl = None):
"""
Changes the priority that's stored in the workload.
Takes the current priority stored in the workload and adds
to it the input priority value.
"""
request = requestDetails(requestName)
# change in Oracle
newPrior = int(priority)
ChangeState.changeRequestPriority(requestName, newPrior)
# change in workload (spec)
helper = loadWorkload(request)
helper.data.request.priority = newPrior
saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
# change priority in CouchDB
couchDb = Database(request["CouchWorkloadDBName"], request["CouchURL"])
fields = {"RequestPriority": newPrior}
couchDb.updateDocument(requestName, "ReqMgr", "updaterequest", fields=fields)
# push the change to the WorkQueue
response = ProdManagement.getProdMgr(requestName)
if response == [] or response[0] is None or response[0] == "":
# Request must not be assigned yet, we are safe here
return
workqueue = WorkQueue.WorkQueue(response[0])
workqueue.updatePriority(requestName, priority)
return
def changePriority(requestName, priority, wmstatUrl = None):
"""
Changes the priority that's stored in the workload.
Takes the current priority stored in the workload and adds
to it the input priority value.
"""
request = requestDetails(requestName)
# change in Oracle
newPrior = int(priority)
ChangeState.changeRequestPriority(requestName, newPrior)
# change in workload (spec)
helper = loadWorkload(request)
helper.data.request.priority = newPrior
saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
# change priority in CouchDB
couchDb = Database(request["CouchWorkloadDBName"], request["CouchURL"])
fields = {"RequestPriority": newPrior}
couchDb.updateDocument(requestName, "ReqMgr", "updaterequest", fields=fields)
# push the change to the WorkQueue
response = ProdManagement.getProdMgr(requestName)
if response == [] or response[0] is None or response[0] == "":
# Request must not be assigned yet, we are safe here
return
workqueue = WorkQueue.WorkQueue(response[0])
workqueue.updatePriority(requestName, priority)
return
def main():
print "Looking for problematic inbox elements..."
problemRequests = getProblematicRequests()
print "Found %d bad elements:" % len(problemRequests)
if not problemRequests:
print "Nothing to fix, contact a developer if the problem persists..."
return 0
for request in problemRequests:
print request["RequestName"]
var = raw_input("Can we close these for new data in inbox elements: Y/N\n")
if var == "Y":
print "Updating them in global inbox, you need a WMAgent proxy for this."
inboxDB = Database('workqueue_inbox', 'https://cmsweb.cern.ch/couchdb')
for request in problemRequests:
inboxDB.document(request._id)
inboxDB.updateDocument(request._id, 'WorkQueue', 'in-place', fields={'OpenForNewData': false})
print "Done with the deletions, this should fix the problem."
return 0
else:
var = raw_input("Then can we delete these inbox elements: Y/N\n")
if var == "Y":
print "Deleting them from the global inbox, you need a WMAgent proxy for this."
inboxDB = Database('workqueue_inbox', 'https://cmsweb.cern.ch/couchdb')
for request in problemRequests:
inboxDB.delete_doc(request._id, request.rev)
print "Done with the deletions, this should fix the problem."
return 0
else:
print "Doing nothing as you commanded..."
return 0
def changeRequestStatus(requestName, newState, priority=None, wmstatUrl=None):
"""
_changeRequestStatus_
Basic API to change a request to a new state, also
includes optional priority change for the request
- *requestName* : name of the request to be modified
- *newState* : name of the new status for the request
- *priority* : optional integer priority
Apparently when changing request state (on assignment page),
it's possible to change priority at one go. Hence the argument is here.
"""
# MySQL/Oracle
factory = DBConnect.getConnection()
reqId = getRequestID(factory, requestName)
changeRequestIDStatus(reqId, newState, priority)
# CouchDB
# have to first get information where the request Couch document is,
# extracting the information from reqmgr_request.workflow table field
reqData = factory(classname="Request.Get").execute(reqId)
# this would be something like this:
# http://localhost:5984/reqmgr_workload_cache/maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
wfUrl = reqData['workflow']
# cut off /maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
couchUrl = wfUrl.replace('/' + requestName + "/spec", '')
couchDbName = couchUrl[couchUrl.rfind('/') + 1:]
# cut off database name from the URL
url = couchUrl.replace('/' + couchDbName, '')
couchDb = Database(couchDbName, url)
fields = {"RequestStatus": newState}
couchDb.updateDocument(requestName,
"ReqMgr",
"updaterequest",
fields=fields,
useBody=True)
#TODO: should we make this mendatory?
if wmstatUrl:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.updateRequestStatus(requestName, newState)
def changeRequestStatus(requestName, newState, priority=None, wmstatUrl=None):
"""
_changeRequestStatus_
Basic API to change a request to a new state, also
includes optional priority change for the request
- *requestName* : name of the request to be modified
- *newState* : name of the new status for the request
- *priority* : optional integer priority
Apparently when changing request state (on assignment page),
it's possible to change priority at one go. Hence the argument is here.
"""
# MySQL/Oracle
factory = DBConnect.getConnection()
reqId = getRequestID(factory, requestName)
changeRequestIDStatus(reqId, newState, priority)
# CouchDB
# have to first get information where the request Couch document is,
# extracting the information from reqmgr_request.workflow table field
reqData = factory(classname="Request.Get").execute(reqId)
# this would be something like this:
# http://localhost:5984/reqmgr_workload_cache/maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
wfUrl = reqData["workflow"]
# cut off /maxa_RequestString-OVERRIDE-ME_130306_205649_8066/spec
couchUrl = wfUrl.replace("/" + requestName + "/spec", "")
couchDbName = couchUrl[couchUrl.rfind("/") + 1 :]
# cut off database name from the URL
url = couchUrl.replace("/" + couchDbName, "")
couchDb = Database(couchDbName, url)
fields = {"RequestStatus": newState}
couchDb.updateDocument(requestName, "ReqMgr", "updaterequest", fields=fields, useBody=True)
# TODO: should we make this mendatory?
if wmstatUrl:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.updateRequestStatus(requestName, newState)
def changePriority(requestName, priority, wmstatUrl = None):
"""
Changes the priority that's stored in the workload.
Takes the current priority stored in the workload and adds
to it the input priority value.
"""
request = requestDetails(requestName)
# change in Oracle
newPrior = int(priority)
ChangeState.changeRequestPriority(requestName, newPrior)
# change in workload (spec)
helper = loadWorkload(request)
helper.data.request.priority = newPrior
saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
# change priority in CouchDB
couchDb = Database(request["CouchWorkloadDBName"], request["CouchURL"])
fields = {"RequestPriority": newPrior}
couchDb.updateDocument(requestName, "ReqMgr", "updaterequest", fields=fields, useBody = True)
# push the change to the WorkQueue
gqURL = "%s/workqueue" % request["CouchURL"]
workqueue = WorkQueue.WorkQueue(gqURL)
workqueue.updatePriority(requestName, priority)
return
# Block closing information
blockCloseMaxWaitTime = int(
kwargs.get("BlockCloseMaxWaitTime",
helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(
kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(
kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(
kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'],
self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
couchDb.updateDocument(
request["RequestName"],
"ReqMgr",
"updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"]})
def updateRequestStats(requestName, stats, couchURL, couchDBName):
couchDB = Database(couchDBName, couchURL)
return couchDB.updateDocument(requestName, 'ReqMgr', 'totalstats',
fields=stats)
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False):
"""
If clone is True, the function is called on a cloned request in which
case no modification of the reqSchema shall happen and should be checked in
as is.
"""
try:
request = buildWorkloadForRequest(typename = reqSchema["RequestType"],
schema = reqSchema)
except WMSpecFactoryException as ex:
logging.error(traceback.format_exc())
raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())
helper = WMWorkloadHelper(request['WorkloadSpec'])
# update request as well for wmstats update
# there is a better way to do this (passing helper to request but make sure all the information is there)
request["Campaign"] = helper.getCampaign()
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
for ds in helper.listOutputDatasets():
if ds not in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
#TODO: need to update output dataset by Task for task chain requests
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema['RequestType'])
except CheckIn.RequestCheckInError as ex:
raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
# Inconsistent request parameters between Oracle and Couch (#4380, #4388)
# metadata above is what is saved into couch to represent a request document.
# Number of request arguments on a corresponding couch document
# is not set, has default null/None values, update those accordingly now.
# It's a mess to have two mutually inconsistent database backends.
# Not easy to handle this earlier since couch is stored first and
# some parameters are worked out later when storing into Oracle.
reqDetails = requestDetails(request["RequestName"])
# couchdb request parameters which are null at the injection time and remain so
paramsToUpdate = ["RequestStatus",
"RequestSizeFiles",
"AcquisitionEra",
"RequestWorkflow",
"RequestType",
"RequestStatus",
"RequestPriority",
"Requestor",
"Group",
"SizePerEvent",
"PrepID",
"RequestNumEvents",
"ProcessingString",
"ProcessingVersion",
]
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
fields = {}
for key in paramsToUpdate:
fields[key] = reqDetails[key]
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields, useBody=True)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400, "Creating request failed, could not update WMStats.")
return request
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
# Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites)
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(
int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)),
)
helper.setupPerformanceMonitoring(
int(kwargs.get("maxRSS", 2411724)),
int(kwargs.get("maxVSize", 2411724)),
int(kwargs.get("SoftTimeout", 129600)),
int(kwargs.get("GracePeriod", 300)),
)
# Check whether we should check location for the data
if "useSiteListAsLocation" in kwargs:
helper.setLocationDataSourceFlag()
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority")
subscriptionType = kwargs.get("CustodialSubType", "Move")
if subscriptionType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid custodial subscription type")
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=subscriptionType,
priority=subscriptionPriority,
)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request["RequestWorkflow"], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(
request["RequestName"], "ReqMgr", "updaterequest", fields={"AcquisitionEra": reqDetails["AcquisitionEra"]}
)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
#Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if type(whiteList) != list:
whiteList = [whiteList]
if type(blackList) != list:
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist = whiteList, siteBlacklist = blackList,
wildcardDict = self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# Now verify the output datasets
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(400,
"Bad output dataset name, check the processed dataset.\n %s" %
str(ex))
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout",None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
useAAA = strToBool(kwargs.get("useSiteListAsLocation", False))
if useAAA:
helper.setLocationDataSourceFlag(flag = useAAA)
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
for site in autoApproveList:
if site.endswith('_MSS'):
raise cherrypy.HTTPError(400, "Auto-approval to MSS endpoint not allowed %s" % autoApproveList)
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority %s" % subscriptionPriority)
custodialType = kwargs.get("CustodialSubType", "Replica")
if custodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % custodialType)
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
if nonCustodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid noncustodial subscription type %s" % nonCustodialType)
helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
custodialSites = custodialList,
nonCustodialSites = nonCustodialList,
autoApproveSites = autoApproveList,
custodialSubType = custodialType,
nonCustodialSubType = nonCustodialType,
priority = subscriptionPriority)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
# set Task properties if they are exist
# TODO: need to define the task format (maybe kwargs["tasks"]?)
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"],
"ProcessingVersion": reqDetails["ProcessingVersion"],
"CustodialSites": custodialList,
"NonCustodialSites": nonCustodialList,
"AutoApproveSubscriptionSites": autoApproveList,
"SubscriptionPriority": subscriptionPriority,
"CustodialSubType": custodialType,
"NonCustodialSubType": nonCustodialType,
"Teams": kwargs["Teams"],
"OutputDatasets": outputDatasets,
"SiteWhitelist": whiteList,
"SiteBlacklist": blackList},
useBody = True)
"RequestWorkflow",
"RequestType",
"RequestStatus",
"RequestPriority",
"Requestor",
"Group",
"SizePerEvent",
"PrepID",
"RequestNumEvents",
]
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
fields = {}
for key in paramsToUpdate:
fields[key] = reqDetails[key]
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400, "Creating request failed, could not update WMStats.")
return request
def makeRequest(webApi, reqInputArgs, couchUrl, couchDB, wmstatUrl):
"""
Handles the submission of requests.
subscriptionType = kwargs.get("CustodialSubType", "Move")
if subscriptionType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % subscriptionType)
helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
custodialSites = custodialList,
nonCustodialSites = nonCustodialList,
autoApproveSites = autoApproveList,
custodialSubType = subscriptionType,
priority = subscriptionPriority)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"]})
blockCloseMaxSize = int(
kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'],
self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"],
"ReqMgr",
"updaterequest",
fields={
"AcquisitionEra":
reqDetails["AcquisitionEra"],
"Teams":
JsonWrapper.JSONEncoder().encode(
kwargs["Teams"]),
"SiteWhitelist":
JsonWrapper.JSONEncoder().encode(whiteList),
"SiteBlacklist":
JsonWrapper.JSONEncoder().encode(blackList)
})
helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
custodialSites = custodialList,
nonCustodialSites = nonCustodialList,
autoApproveSites = autoApproveList,
custodialSubType = subscriptionType,
priority = subscriptionPriority)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"],
"Teams": JsonWrapper.JSONEncoder().encode(kwargs["Teams"]),
"SiteWhitelist": JsonWrapper.JSONEncoder().encode(whiteList),
"SiteBlacklist": JsonWrapper.JSONEncoder().encode(blackList)})
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
try:
helper.validateArgumentForAssignment(kwargs)
except WMSpecFactoryException as ex:
raise cherrypy.HTTPError(400, str(ex.message()))
except Exception:
msg = traceback.format_exc()
raise cherrypy.HTTPError(400, "Unhandled error: %s" % msg)
# Validate the different parts of the processed dataset
processedDatasetParts = {"AcquisitionEra": kwargs.get("AcquisitionEra"),
"ProcessingString": kwargs.get("ProcessingString"),
"ProcessingVersion": kwargs.get("ProcessingVersion")
}
for field, values in processedDatasetParts.iteritems():
if field in kwargs and isinstance(kwargs[field], dict):
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs.get(field, values), field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if not isinstance(whiteList, list):
whiteList = [whiteList]
if not isinstance(blackList, list):
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
helper.setAcquisitionEra(kwargs.get("AcquisitionEra", None))
helper.setProcessingString(kwargs.get("ProcessingString", None))
helper.setProcessingVersion(kwargs.get("ProcessingVersion", None))
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(400,
"Bad output dataset name, check the processed dataset.\n %s" %
str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
# FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
helper.setTrustLocationFlag(inputFlag=strToBool(kwargs.get("TrustSitelists", False)),
pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False)))
helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False)))
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
custodialType = kwargs.get("CustodialSubType", "Replica")
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
helper.setSubscriptionInformationWildCards(wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
custodialGroup=kwargs.get("CustodialGroup", "DataOps"),
nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"),
priority=subscriptionPriority,
deleteFromSource=kwargs.get("DeleteFromSource", False))
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setMemory(kwargs.get("Memory"))
helper.setCores(kwargs.get("Multicore"))
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"],
"ProcessingVersion": reqDetails["ProcessingVersion"],
"CustodialSites": custodialList,
"NonCustodialSites": nonCustodialList,
"AutoApproveSubscriptionSites": autoApproveList,
"SubscriptionPriority": subscriptionPriority,
"CustodialSubType": custodialType,
"NonCustodialSubType": nonCustodialType,
"CustodialGroup": kwargs.get("CustodialGroup", "DataOps"),
"NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"),
"DeleteFromSource": kwargs.get("DeleteFromSource", False),
"Teams": kwargs["Teams"],
"OutputDatasets": outputDatasets,
"SiteWhitelist": whiteList,
"SiteBlacklist": blackList,
"MergedLFNBase": kwargs["MergedLFNBase"],
"UnmergedLFNBase": kwargs["UnmergedLFNBase"],
"Dashboard": kwargs.get("Dashboard", ""),
"TrustSitelists": kwargs.get("TrustSitelists", False),
"TrustPUSitelists": kwargs.get("TrustPUSitelists", False),
"AllowOpportunistic": kwargs.get("AllowOpportunistic", False)},
useBody=True)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
#Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if type(whiteList) != list:
whiteList = [whiteList]
if type(blackList) != list:
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList,
siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(
400, "White and blacklist the same site is not allowed %s" %
list(res))
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(
400,
"Bad output dataset name, check the processed dataset.\n %s"
% str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
useAAA = strToBool(kwargs.get("useSiteListAsLocation", False))
if useAAA:
helper.setLocationDataSourceFlag(flag=useAAA)
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
for site in autoApproveList:
if site.endswith('_MSS'):
raise cherrypy.HTTPError(
400, "Auto-approval to MSS endpoint not allowed %s" %
autoApproveList)
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(
400, "Invalid subscription priority %s" % subscriptionPriority)
custodialType = kwargs.get("CustodialSubType", "Replica")
if custodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(
400, "Invalid custodial subscription type %s" % custodialType)
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
if nonCustodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(
400,
"Invalid noncustodial subscription type %s" % nonCustodialType)
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
priority=subscriptionPriority)
# Block closing information
blockCloseMaxWaitTime = int(
kwargs.get("BlockCloseMaxWaitTime",
helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(
kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(
kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(
kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
# set Task properties if they are exist
# TODO: need to define the task format (maybe kwargs["tasks"]?)
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'],
self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"],
"ReqMgr",
"updaterequest",
fields={
"AcquisitionEra":
reqDetails["AcquisitionEra"],
"ProcessingVersion":
reqDetails["ProcessingVersion"],
"CustodialSites":
custodialList,
"NonCustodialSites":
nonCustodialList,
"AutoApproveSubscriptionSites":
autoApproveList,
"SubscriptionPriority":
subscriptionPriority,
"CustodialSubType":
custodialType,
"NonCustodialSubType":
nonCustodialType,
"Teams":
kwargs["Teams"],
"OutputDatasets":
outputDatasets,
"SiteWhitelist":
whiteList,
"SiteBlacklist":
blackList
},
useBody=True)
def updateRequestStats(requestName, stats, couchURL, couchDBName):
couchDB = Database(couchDBName, couchURL)
return couchDB.updateDocument(requestName, 'ReqMgr', 'totalstats',
fields=stats)
def buildWorkloadAndCheckIn(webApi,
reqSchema,
couchUrl,
couchDB,
wmstatUrl,
clone=False):
"""
If clone is True, the function is called on a cloned request in which
case no modification of the reqSchema shall happen and should be checked in
as is.
"""
try:
request = buildWorkloadForRequest(typename=reqSchema["RequestType"],
schema=reqSchema)
except WMSpecFactoryException as ex:
logging.error(traceback.format_exc())
raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())
helper = WMWorkloadHelper(request['WorkloadSpec'])
# update request as well for wmstats update
# there is a better way to do this (passing helper to request but make sure all the information is there)
request["Campaign"] = helper.getCampaign()
# Add the output datasets if necessary
# for some bizarre reason OutpuDatasets is list of lists, when cloning
# [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
# #3743
if not clone:
for ds in helper.listOutputDatasets():
if ds not in request['OutputDatasets']:
request['OutputDatasets'].append(ds)
#TODO: need to update output dataset by Task for task chain requests
# can't save Request object directly, because it makes it hard to retrieve the _rev
metadata = {}
metadata.update(request)
# don't want to JSONify the whole workflow
del metadata['WorkloadSpec']
workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
try:
CheckIn.checkIn(request, reqSchema['RequestType'])
except CheckIn.RequestCheckInError as ex:
raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
# Inconsistent request parameters between Oracle and Couch (#4380, #4388)
# metadata above is what is saved into couch to represent a request document.
# Number of request arguments on a corresponding couch document
# is not set, has default null/None values, update those accordingly now.
# It's a mess to have two mutually inconsistent database backends.
# Not easy to handle this earlier since couch is stored first and
# some parameters are worked out later when storing into Oracle.
reqDetails = requestDetails(request["RequestName"])
# couchdb request parameters which are null at the injection time and remain so
paramsToUpdate = [
"RequestStatus",
"RequestSizeFiles",
"AcquisitionEra",
"RequestWorkflow",
"RequestType",
"RequestStatus",
"RequestPriority",
"Requestor",
"Group",
"SizePerEvent",
"PrepID",
"RequestNumEvents",
"ProcessingString",
"ProcessingVersion",
]
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
fields = {}
for key in paramsToUpdate:
fields[key] = reqDetails[key]
couchDb.updateDocument(request["RequestName"],
"ReqMgr",
"updaterequest",
fields=fields,
useBody=True)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400,
"Creating request failed, could not update WMStats.")
return request
"RequestWorkflow",
"RequestType",
"RequestStatus",
"RequestPriority",
"Requestor",
"Group",
"SizePerEvent",
"PrepID",
"RequestNumEvents",
]
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
fields = {}
for key in paramsToUpdate:
fields[key] = reqDetails[key]
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields)
try:
wmstatSvc = WMStatsWriter(wmstatUrl)
wmstatSvc.insertRequest(request)
except Exception as ex:
webApi.error("Could not update WMStats, reason: %s" % ex)
raise HTTPError(400, "Creating request failed, could not update WMStats.")
return request
def makeRequest(webApi, reqInputArgs, couchUrl, couchDB, wmstatUrl):
"""
Handles the submission of requests.
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
# Validate the different parts of the processed dataset
processedDatasetParts = {
"AcquisitionEra": helper.getAcquisitionEra(),
"ProcessingString": helper.getProcessingString(),
"ProcessingVersion": helper.getProcessingVersion(),
}
for field, origValue in processedDatasetParts.iteritems():
if field in kwargs and isinstance(kwargs[field], dict):
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs.get(field, origValue))
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if not isinstance(whiteList, list):
whiteList = [whiteList]
if not isinstance(blackList, list):
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
# Set AcquisitionEra, ProcessingString and ProcessingVersion
# which could be json encoded dicts
if "AcquisitionEra" in kwargs:
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
if "ProcessingString" in kwargs:
helper.setProcessingString(kwargs["ProcessingString"])
if "ProcessingVersion" in kwargs:
helper.setProcessingVersion(kwargs["ProcessingVersion"])
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(400, "Bad output dataset name, check the processed dataset.\n %s" % str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
# FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(
int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)),
)
helper.setupPerformanceMonitoring(
kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None),
)
# Check whether we should check location for the data
helper.setTrustLocationFlag(
inputFlag=strToBool(kwargs.get("TrustSitelists", False)),
pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False)),
)
helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False)))
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
for site in autoApproveList:
if site.endswith("_MSS"):
raise cherrypy.HTTPError(400, "Auto-approval to MSS endpoint not allowed %s" % autoApproveList)
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority %s" % subscriptionPriority)
custodialType = kwargs.get("CustodialSubType", "Replica")
if custodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % custodialType)
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
if nonCustodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid noncustodial subscription type %s" % nonCustodialType)
if "CustodialGroup" in kwargs and not isinstance(kwargs["CustodialGroup"], basestring):
raise cherrypy.HTTPError(400, "Invalid CustodialGroup format %s" % kwargs["CustodialGroup"])
if "NonCustodialGroup" in kwargs and not isinstance(kwargs["NonCustodialGroup"], basestring):
raise cherrypy.HTTPError(400, "Invalid NonCustodialGroup format %s" % kwargs["NonCustodialGroup"])
if "DeleteFromSource" in kwargs and not isinstance(kwargs["DeleteFromSource"], bool):
raise cherrypy.HTTPError(400, "Invalid DeleteFromSource format %s" % kwargs["DeleteFromSource"])
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
custodialGroup=kwargs.get("CustodialGroup", "DataOps"),
nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"),
priority=subscriptionPriority,
deleteFromSource=kwargs.get("DeleteFromSource", False),
)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize)
helper.setMemoryAndCores(kwargs.get("Memory"), kwargs.get("Multicore"))
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request["RequestWorkflow"], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(
request["RequestName"],
"ReqMgr",
"updaterequest",
fields={
"AcquisitionEra": reqDetails["AcquisitionEra"],
"ProcessingVersion": reqDetails["ProcessingVersion"],
"CustodialSites": custodialList,
"NonCustodialSites": nonCustodialList,
"AutoApproveSubscriptionSites": autoApproveList,
"SubscriptionPriority": subscriptionPriority,
"CustodialSubType": custodialType,
"NonCustodialSubType": nonCustodialType,
"CustodialGroup": kwargs.get("CustodialGroup", "DataOps"),
"NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"),
"DeleteFromSource": kwargs.get("DeleteFromSource", False),
"Teams": kwargs["Teams"],
"OutputDatasets": outputDatasets,
"SiteWhitelist": whiteList,
"SiteBlacklist": blackList,
"MergedLFNBase": kwargs["MergedLFNBase"],
"UnmergedLFNBase": kwargs["UnmergedLFNBase"],
"Dashboard": kwargs.get("Dashboard", ""),
"TrustSitelists": kwargs.get("TrustSitelists", False),
"TrustPUSitelists": kwargs.get("TrustPUSitelists", False),
"AllowOpportunistic": kwargs.get("AllowOpportunistic", False),
},
useBody=True,
)
def main():
if len(sys.argv) < 2:
print("Missing the connect Oracle TNS argument (user/password@server).")
sys.exit(1)
tns = sys.argv[1]
print("Creating CouchDB database connection ...")
couchdb = Database(couchdb_name, couch_url)
print("Creating Oracle database connection ...")
oradb = cx_Oracle.Connection(tns)
num_couch_requests = get_couchdb_row_count(couchdb)
print("Total CouchDB request documents in ReqMgr: %s" % num_couch_requests)
num_oracle_requests = get_oracle_row_count(oradb, "reqmgr_request")
print("Total Oracle requests entries in ReqMgr: %s" % num_oracle_requests)
if num_couch_requests != num_oracle_requests:
print("Number of requests in Oracle, CouchDB don't agree, fix that first.")
sys.exit(1)
else:
print("Database cross-check (Oracle request names vs CouchDB): DONE, THE SAME.")
def get_couch_value(couch_req, mapping):
try:
c = couch_req[mapping["couch"]]
couch_missing = False
except KeyError:
# comparison will not happen due to missing flag, the value
# will be stored in couch
c = "N/A"
couch_missing = False
return str(c), couch_missing
def check_oracle_worflow_value(oracle_value, mapping, req_name):
# check Oracle WORKFLOW value
if mapping["oracle"] == "WORKFLOW":
# https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/linacre_2011A_442p2_DataReprocessingMuOnia_111119_005717/spec
from_wf_url_req_name = oracle_value.rsplit('/', 2)[-2]
if req_name != from_wf_url_req_name:
print("Workflow URL mismatch: %s" % o)
sys.exit(1)
counter = 0
for oracle_req in get_oracle_data(oradb):
req_name = oracle_req["REQUEST_NAME"]
# FILTER
# check only requests injected approx. after last deployment (a lot of
# stuff should have already been fixed in ReqMgr)
# _13041._*$ (ending of request name with date/time)
#if not re.match(".*_1304[0-3][0-9]_.*$", req_name): # all April 2013
# continue
counter += 1
print("\n\n%s (%s)" % (req_name, counter))
couch_req = couchdb.document(req_name)
couch_fields_to_correct = {}
for mapping in MAPPING:
if mapping["couch"] in COUCH_TO_IGNORE:
continue
o = str(oracle_req[mapping["oracle"]])
c, couch_missing = get_couch_value(couch_req, mapping)
check_oracle_worflow_value(o, mapping, req_name)
# compare oracle and couch values
# don't update value in couch if it exists and is non-empty
if (couch_missing or o != c) and c in ('None', '0', '', "N/A"):
print("%s %s != %s" % (mapping, o, c))
# correct couch request by oracle value
couch_fields_to_correct[mapping["couch"]] = o
if couch_fields_to_correct:
print("Couch corrected fields:")
print(couch_fields_to_correct)
if sys.argv[-1] == "-c":
couchdb.updateDocument(req_name, "ReqMgr", "updaterequest",
fields=couch_fields_to_correct, useBody=True)
print("Couch updated")
else:
print("OK")
# fields that should be removed from couch
"""
