def getOutputForRequest(requestName):
"""Return the datasets produced by this request."""
request = GetRequest.getRequestByName(requestName)
if not request:
return []
helper = loadWorkload(request)
return helper.listOutputDatasets()
def makeWorkload(self, schema):
originalRequest = GetRequest.getRequestByName(schema['OriginalRequestName'])
helper = loadWorkload(originalRequest)
helper.truncate(schema["RequestName"], schema["InitialTaskPath"],
schema["ACDCServer"], schema["ACDCDatabase"])
return helper.data
def changeStatus(requestName, status, wmstatUrl, acdcUrl):
""" Changes the status for this request """
request = GetRequest.getRequestByName(requestName)
if not status in RequestStatus.StatusList:
raise RuntimeError, "Bad status code " + status
if not request.has_key('RequestStatus'):
raise RuntimeError, "Cannot find status for request " + requestName
oldStatus = request['RequestStatus']
if not status in RequestStatus.NextStatus[oldStatus]:
raise RuntimeError, "Cannot change status from %s to %s. Allowed values are %s" % (
oldStatus, status, RequestStatus.NextStatus[oldStatus])
if status == 'aborted':
# delete from the workqueue
if not privileged() and not ownsRequest(request):
raise cherrypy.HTTPError(403, "You are not allowed to abort this request")
elif not privileged():
raise cherrypy.HTTPError(403, "You are not allowed to change the state for this request")
# delete from the workqueue if it's been assigned to one
if oldStatus in ["acquired", "running", "running-closed", "running-open"]:
abortRequest(requestName)
else:
raise cherrypy.HTTPError(400, "You cannot abort a request in state %s" % oldStatus)
if status == 'announced':
# cleanup acdc database, if possible
if acdcUrl:
url, database = WMCore.Lexicon.splitCouchServiceURL(acdcUrl)
acdcService = CouchService(url = url, database = database)
acdcService.removeFilesetsByCollectionName(requestName)
# finally, perform the transition, have to do it in both Oracle and CouchDB
# and in WMStats
ChangeState.changeRequestStatus(requestName, status, wmstatUrl=wmstatUrl)
def splitting(self, requestName):
"""
_splitting_
Retrieve the current values for splitting parameters for all tasks in
the spec. Format them in the manner that the splitting page expects
and pass them to the template.
"""
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
splittingDict = helper.listJobSplittingParametersByTask()
taskNames = splittingDict.keys()
taskNames.sort()
splitInfo = []
for taskName in taskNames:
jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName])
splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"],
"splitParams": jsonSplittingParams,
"taskType": splittingDict[taskName]["type"],
"taskName": taskName})
return self.templatepage("Splitting", requestName = requestName,
taskInfo = splitInfo, taskNames = taskNames)
def getOutputForRequest(requestName):
"""Return the datasets produced by this request."""
request = GetRequest.getRequestByName(requestName)
if not request:
return []
helper = loadWorkload(request)
return helper.listOutputDatasets()
def one(self, requestName):
""" Assign a single request """
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
request = Utilities.prepareForTable(request)
requestType = request["RequestType"]
# get assignments
teams = ProdManagement.listTeams()
assignments = GetRequest.getAssignmentsByName(requestName)
# might be a list, or a dict team:priority
if isinstance(assignments, dict):
assignments = assignments.keys()
procVer = ""
acqEra = ""
helper = Utilities.loadWorkload(request)
if helper.getAcquisitionEra() != None:
acqEra = helper.getAcquisitionEra()
if helper.getProcessingVersion() != None:
procVer = helper.getProcessingVersion()
dashboardActivity = helper.getDashboardActivity()
(reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
return self.templatepage("Assign", requests=[request], teams=teams,
assignments=assignments, sites=self.sites,
mergedLFNBases=self.mergedLFNBases[requestType],
reqMergedBase=reqMergedBase,
unmergedLFNBases=self.allUnmergedLFNBases,
reqUnmergedBase=reqUnmergedBase,
acqEra = acqEra, procVer = procVer,
dashboardActivity=dashboardActivity,
badRequests=[])
def changeStatus(requestName, status, wmstatUrl, acdcUrl):
""" Changes the status for this request """
request = GetRequest.getRequestByName(requestName)
if not status in RequestStatus.StatusList:
raise RuntimeError, "Bad status code " + status
if not request.has_key('RequestStatus'):
raise RuntimeError, "Cannot find status for request " + requestName
oldStatus = request['RequestStatus']
if not status in RequestStatus.NextStatus[oldStatus]:
raise RuntimeError, "Cannot change status from %s to %s. Allowed values are %s" % (
oldStatus, status, RequestStatus.NextStatus[oldStatus])
if status == 'aborted' or status == 'force-complete':
# delete from the workqueue
if not privileged() and not ownsRequest(request):
raise cherrypy.HTTPError(403, "You are not allowed to %s this request" % status)
elif not privileged():
raise cherrypy.HTTPError(403, "You are not allowed to change the state for this request")
# delete from the workqueue if it's been assigned to one
if status in RequestStatus.NextStatus[oldStatus]:
abortRequest(requestName)
else:
raise cherrypy.HTTPError(400, "You cannot abort a request in state %s" % oldStatus)
if status == 'announced':
# cleanup acdc database, if possible
if acdcUrl:
url, database = WMCore.Lexicon.splitCouchServiceURL(acdcUrl)
acdcService = CouchService(url = url, database = database)
acdcService.removeFilesetsByCollectionName(requestName)
# finally, perform the transition, have to do it in both Oracle and CouchDB
# and in WMStats
ChangeState.changeRequestStatus(requestName, status, wmstatUrl=wmstatUrl)
def changeStatus(requestName, status, wmstatUrl, acdcUrl):
""" Changes the status for this request """
request = GetRequest.getRequestByName(requestName)
if not status in RequestStatus.StatusList:
raise RuntimeError("Bad status code " + status)
if 'RequestStatus' not in request:
raise RuntimeError("Cannot find status for request " + requestName)
oldStatus = request['RequestStatus']
if not status in RequestStatus.NextStatus[oldStatus]:
raise RuntimeError("Cannot change status from %s to %s. Allowed values are %s" % (
oldStatus, status, RequestStatus.NextStatus[oldStatus]))
if status == 'aborted' or status == 'force-complete':
# delete from the workqueue
if not privileged() and not ownsRequest(request):
raise cherrypy.HTTPError(403, "You are not allowed to %s this request" % status)
elif not privileged():
raise cherrypy.HTTPError(403, "You are not allowed to change the state for this request")
# delete from the workqueue if it's been assigned to one
if status in RequestStatus.NextStatus[oldStatus]:
abortRequest(requestName)
else:
raise cherrypy.HTTPError(400, "You cannot abort a request in state %s" % oldStatus)
# ACDC database is cleaned by separate thread
# if status == 'announced':
# # cleanup acdc database, if possible
# if acdcUrl:
# url, database = WMCore.Lexicon.splitCouchServiceURL(acdcUrl)
# acdcService = CouchService(url = url, database = database)
# acdcService.removeFilesetsByCollectionName(requestName)
# finally, perform the transition, have to do it in both Oracle and CouchDB
# and in WMStats
ChangeState.changeRequestStatus(requestName, status, wmstatUrl=wmstatUrl)
def splitting(self, requestName):
"""
_splitting_
Retrieve the current values for splitting parameters for all tasks in
the spec. Format them in the manner that the splitting page expects
and pass them to the template.
"""
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
splittingDict = helper.listJobSplittingParametersByTask(performance = False)
taskNames = splittingDict.keys()
taskNames.sort()
splitInfo = []
for taskName in taskNames:
jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName])
splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"],
"splitParams": jsonSplittingParams,
"taskType": splittingDict[taskName]["type"],
"taskName": taskName})
return self.templatepage("Splitting", requestName = requestName,
taskInfo = splitInfo, taskNames = taskNames)
def changeStatus(requestName, status, wmstatUrl):
""" Changes the status for this request """
request = GetRequest.getRequestByName(requestName)
if not status in RequestStatus.StatusList:
raise RuntimeError, "Bad status code " + status
if not request.has_key('RequestStatus'):
raise RuntimeError, "Cannot find status for request " + requestName
oldStatus = request['RequestStatus']
if not status in RequestStatus.NextStatus[oldStatus]:
raise RuntimeError, "Cannot change status from %s to %s. Allowed values are %s" % (
oldStatus, status, RequestStatus.NextStatus[oldStatus])
if status == 'aborted':
# delete from the workqueue
if not privileged() and not ownsRequest(request):
raise cherrypy.HTTPError(
403, "You are not allowed to abort this request")
elif not privileged():
raise cherrypy.HTTPError(
403,
"You are not allowed to change the state for this request")
# delete from the workqueue if it's been assigned to one
if oldStatus in ["acquired", "running"]:
abortRequest(requestName)
else:
raise cherrypy.HTTPError(
400, "You cannot abort a request in state %s" % oldStatus)
#FIXME needs logic about who is allowed to do which transition
ChangeState.changeRequestStatus(requestName, status, wmstatUrl=wmstatUrl)
def changeStatus(requestName, status, wmstatUrl):
""" Changes the status for this request """
request = GetRequest.getRequestByName(requestName)
if not status in RequestStatus.StatusList:
raise RuntimeError, "Bad status code " + status
if not request.has_key('RequestStatus'):
raise RuntimeError, "Cannot find status for request " + requestName
oldStatus = request['RequestStatus']
if not status in RequestStatus.NextStatus[oldStatus]:
raise RuntimeError, "Cannot change status from %s to %s. Allowed values are %s" % (
oldStatus, status, RequestStatus.NextStatus[oldStatus])
if status == 'aborted':
# delete from the workqueue
if not privileged() and not ownsRequest(request):
raise cherrypy.HTTPError(403, "You are not allowed to abort this request")
elif not privileged():
raise cherrypy.HTTPError(403, "You are not allowed to change the state for this request")
# delete from the workqueue if it's been assigned to one
if oldStatus in ["acquired", "running"]:
abortRequest(requestName)
else:
raise cherrypy.HTTPError(400, "You cannot abort a request in state %s" % oldStatus)
#FIXME needs logic about who is allowed to do which transition
ChangeState.changeRequestStatus(requestName, status, wmstatUrl = wmstatUrl)
return
def makeWorkload(self, schema):
originalRequest = GetRequest.getRequestByName(
schema['OriginalRequestName'])
helper = loadWorkload(originalRequest)
helper.truncate(schema["RequestName"], schema["InitialTaskPath"],
schema["ACDCServer"], schema["ACDCDatabase"])
return helper.data
def __call__(self, workloadName, arguments):
requestName = arguments['OriginalRequestName']
originalRequest = GetRequest.getRequestByName(requestName)
helper = loadWorkload(originalRequest)
helper.truncate(arguments["RequestName"], arguments["InitialTaskPath"],
arguments["ACDCServer"], arguments["ACDCDatabase"])
return helper
def changePriority(requestName, priority):
""" Changes the priority that's stored in the workload """
# fill in all details
request = GetRequest.getRequestByName(requestName)
ChangeState.changeRequestPriority(requestName, priority)
helper = loadWorkload(request)
helper.data.request.priority = int(priority)
saveWorkload(helper, request['RequestWorkflow'])
def __call__(self, workloadName, arguments):
requestName = arguments['OriginalRequestName']
originalRequest = GetRequest.getRequestByName(requestName)
helper = loadWorkload(originalRequest)
helper.truncate(arguments["RequestName"], arguments["InitialTaskPath"],
arguments["ACDCServer"], arguments["ACDCDatabase"],
arguments.get("CollectionName"))
helper.ignoreOutputModules(arguments.get("IgnoredOutputModules", []))
return helper
def __call__(self, workloadName, arguments):
requestName = arguments['OriginalRequestName']
originalRequest = GetRequest.getRequestByName(requestName)
helper = loadWorkload(originalRequest)
helper.truncate(arguments["RequestName"], arguments["InitialTaskPath"],
arguments["ACDCServer"], arguments["ACDCDatabase"],
arguments.get("CollectionName"))
helper.ignoreOutputModules(arguments.get("IgnoredOutputModules", []))
return helper
def handleSplittingPage(self, requestName, splittingTask, splittingAlgo,
**submittedParams):
"""
_handleSplittingPage_
Parse job splitting parameters sent from the splitting parameter update
page. Pull down the request and modify the new spec applying the
updated splitting parameters.
"""
splitParams = {}
if splittingAlgo == "FileBased":
splitParams["files_per_job"] = int(submittedParams["files_per_job"])
elif splittingAlgo == "TwoFileBased":
splitParams["files_per_job"] = int(submittedParams["two_files_per_job"])
elif splittingAlgo == "LumiBased":
splitParams["lumis_per_job"] = int(submittedParams["lumis_per_job"])
if str(submittedParams["halt_job_on_file_boundaries"]) == "True":
splitParams["halt_job_on_file_boundaries"] = True
else:
splitParams["halt_job_on_file_boundaries"] = False
elif splittingAlgo == "EventAwareLumiBased":
splitParams["events_per_job"] = int(submittedParams["avg_events_per_job"])
splitParams["max_events_per_lumi"] = int(submittedParams["max_events_per_lumi"])
if str(submittedParams["halt_job_on_file_boundaries_event_aware"]) == "True":
splitParams["halt_job_on_file_boundaries"] = True
else:
splitParams["halt_job_on_file_boundaries"] = False
elif splittingAlgo == "EventBased":
splitParams["events_per_job"] = int(submittedParams["events_per_job"])
if "events_per_lumi" in submittedParams:
splitParams["events_per_lumi"] = int(submittedParams["events_per_lumi"])
if "lheInputFiles" in submittedParams:
if str(submittedParams["lheInputFiles"]) == "True":
splitParams["lheInputFiles"] = True
else:
splitParams["lheInputFiles"] = False
elif splittingAlgo == "Harvest":
splitParams["periodic_harvest_interval"] = int(submittedParams["periodic_harvest_interval"])
elif 'Merg' in splittingTask:
for field in ['min_merge_size', 'max_merge_size', 'max_merge_events', 'max_wait_time']:
splitParams[field] = int(submittedParams[field])
if "include_parents" in submittedParams.keys():
if str(submittedParams["include_parents"]) == "True":
splitParams["include_parents"] = True
else:
splitParams["include_parents"] = False
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
logging.info("SetSplitting " + requestName + splittingTask + splittingAlgo + str(splitParams))
helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams)
Utilities.saveWorkload(helper, request['RequestWorkflow'])
return "Successfully updated splitting parameters for " + splittingTask \
+ " " + detailsBackLink(requestName)
def changePriority(requestName, priority, wmstatUrl = None):
""" Changes the priority that's stored in the workload """
# fill in all details
request = GetRequest.getRequestByName(requestName)
groupPriority = request.get('ReqMgrGroupBasePriority', 0)
userPriority = request.get('ReqMgrRequestorBasePriority', 0)
ChangeState.changeRequestPriority(requestName, priority)
helper = loadWorkload(request)
totalPriority = int(priority) + int(userPriority) + int(groupPriority)
helper.data.request.priority = totalPriority
saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
def changePriority(requestName, priority, wmstatUrl=None):
""" Changes the priority that's stored in the workload """
# fill in all details
request = GetRequest.getRequestByName(requestName)
groupPriority = request.get('ReqMgrGroupBasePriority', 0)
userPriority = request.get('ReqMgrRequestorBasePriority', 0)
ChangeState.changeRequestPriority(requestName, priority)
helper = loadWorkload(request)
totalPriority = int(priority) + int(userPriority) + int(groupPriority)
helper.data.request.priority = totalPriority
saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
for field in ["AcquisitionEra", "ProcessingVersion"]:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
helper.setSiteWildcardsLists(siteWhitelist=whiteList,
siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(
int(kwargs.get("maxRSS", 2411724)),
int(kwargs.get("maxVSize", 2411724)),
int(kwargs.get("SoftTimeout", 167000)),
int(kwargs.get("GracePeriod", 300)))
# Check whether we should check location for the data
if "useSiteListAsLocation" in kwargs:
helper.setLocationDataSourceFlag()
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
if "AutoApprove" in kwargs:
autoApproveList = nonCustodialList
else:
autoApproveList = []
priority = kwargs.get("Priority", "Low")
if priority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority")
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
priority=priority)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'],
self.wmstatWriteURL)
def showWorkload(self, requestName):
""" Displays the workload """
self.validate(requestName)
try:
request = GetRequest.getRequestByName(requestName)
except (Exception, RuntimeError) as ex:
raise cherrypy.HTTPError(400, "Invalid request.")
request = Utilities.prepareForTable(request)
helper = Utilities.loadWorkload(request)
workloadText = str(helper.data)
return cgi.escape(workloadText).replace("\n", "<br/>\n")
def showWorkload(self, requestName):
""" Displays the workload """
self.validate(requestName)
try:
request = GetRequest.getRequestByName(requestName)
except (Exception, RuntimeError) as ex:
raise cherrypy.HTTPError(400, "Invalid request. %s" % str(ex))
request = Utilities.prepareForTable(request)
helper = Utilities.loadWorkload(request)
workloadText = str(helper.data)
return cgi.escape(workloadText).replace("\n", "<br/>\n")
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
for field in ["AcquisitionEra", "ProcessingVersion"]:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
helper.setSiteWildcardsLists(siteWhitelist = whiteList, siteBlacklist = blackList,
wildcardDict = self.wildcardSites)
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(int(kwargs.get("maxRSS", 2411724)),
int(kwargs.get("maxVSize", 2411724)),
int(kwargs.get("SoftTimeout", 171600)),
int(kwargs.get("GracePeriod", 300)))
# Check whether we should check location for the data
if "useSiteListAsLocation" in kwargs:
helper.setLocationDataSourceFlag()
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
if "AutoApprove" in kwargs:
autoApproveList = nonCustodialList
else:
autoApproveList = []
priority = kwargs.get("Priority", "Low")
if priority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority")
helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
custodialSites = custodialList,
nonCustodialSites = nonCustodialList,
autoApproveSites = autoApproveList,
priority = priority)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
def one(self, requestName):
""" Assign a single request """
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
request = Utilities.prepareForTable(request)
# get assignments
teams = ProdManagement.listTeams()
assignments = GetRequest.getAssignmentsByName(requestName)
# might be a list, or a dict team:priority
if isinstance(assignments, dict):
assignments = assignments.keys()
procVer = ""
acqEra = ""
procString = ""
helper = Utilities.loadWorkload(request)
if helper.getAcquisitionEra() != None:
acqEra = helper.getAcquisitionEra()
if helper.getProcessingVersion() != None:
procVer = helper.getProcessingVersion()
if helper.getProcessingString():
procString = helper.getProcessingString()
dashboardActivity = helper.getDashboardActivity()
blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime()
blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
blockCloseMaxSize = helper.getBlockCloseMaxSize()
(reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
return self.templatepage(
"Assign",
requests=[request],
teams=teams,
assignments=assignments,
sites=self.sites,
phedexNodes=self.phedexNodes,
mergedLFNBases=self.allMergedLFNBases,
reqMergedBase=reqMergedBase,
unmergedLFNBases=self.allUnmergedLFNBases,
reqUnmergedBase=reqUnmergedBase,
acqEra=acqEra,
procVer=procVer,
procString=procString,
dashboardActivity=dashboardActivity,
badRequests=[],
blockCloseMaxWaitTime=blockCloseMaxWaitTime,
blockCloseMaxFiles=blockCloseMaxFiles,
blockCloseMaxSize=blockCloseMaxSize,
blockCloseMaxEvents=blockCloseMaxEvents,
)
def one(self, requestName):
""" Assign a single request """
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
request = Utilities.prepareForTable(request)
requestType = request["RequestType"]
# get assignments
teams = ProdManagement.listTeams()
assignments = GetRequest.getAssignmentsByName(requestName)
# might be a list, or a dict team:priority
if isinstance(assignments, dict):
assignments = assignments.keys()
procVer = ""
acqEra = ""
procString = ""
helper = Utilities.loadWorkload(request)
if helper.getAcquisitionEra() != None:
acqEra = helper.getAcquisitionEra()
if helper.getProcessingVersion() != None:
procVer = helper.getProcessingVersion()
if helper.getProcessingString():
procString = helper.getProcessingString()
dashboardActivity = helper.getDashboardActivity()
blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime()
blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
blockCloseMaxSize = helper.getBlockCloseMaxSize()
(reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
return self.templatepage("Assign",
requests=[request],
teams=teams,
assignments=assignments,
sites=self.sites,
phedexNodes=self.phedexNodes,
mergedLFNBases=self.allMergedLFNBases,
reqMergedBase=reqMergedBase,
unmergedLFNBases=self.allUnmergedLFNBases,
reqUnmergedBase=reqUnmergedBase,
acqEra=acqEra,
procVer=procVer,
procString=procString,
dashboardActivity=dashboardActivity,
badRequests=[],
blockCloseMaxWaitTime=blockCloseMaxWaitTime,
blockCloseMaxFiles=blockCloseMaxFiles,
blockCloseMaxSize=blockCloseMaxSize,
blockCloseMaxEvents=blockCloseMaxEvents)
def changeStatus(requestName, status):
""" Changes the status for this request """
request = GetRequest.getRequestByName(requestName)
oldStatus = request['RequestStatus']
if not status in RequestStatus.StatusList:
raise RuntimeError, "Bad status code " + status
if not request.has_key('RequestStatus'):
raise RuntimeError, "Cannot find status for request " + requestName
if not status in RequestStatus.NextStatus[oldStatus]:
raise RuntimeError, "Cannot change status from %s to %s. Allowed values are %s" % (
oldStatus, status, RequestStatus.NextStatus[oldStatus])
ChangeState.changeRequestStatus(requestName, status)
if status == 'aborted':
# delete from the workqueue
abortRequest(requestName)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
#Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if type(whiteList) != list:
whiteList = [whiteList]
if type(blackList) != list:
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList,
siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# Now verify the output datasets
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError, ex:
raise cherrypy.HTTPError(
400,
"Bad output dataset name, check the processed dataset.\n %s"
% str(ex))
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
for field in ["AcquisitionEra", "ProcessingVersion"]:
self.validate(kwargs[field], field)
helper.setSiteWhitelist(Utilities.parseSite(kwargs,"SiteWhitelist"))
helper.setSiteBlacklist(Utilities.parseSite(kwargs,"SiteBlacklist"))
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs["MinMergeSize"]),
int(kwargs["MaxMergeSize"]),
int(kwargs["MaxMergeEvents"]))
helper.setupPerformanceMonitoring(int(kwargs["maxRSS"]),
int(kwargs["maxVSize"]))
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'])
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
#Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if type(whiteList) != list:
whiteList = [whiteList]
if type(blackList) != list:
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist = whiteList, siteBlacklist = blackList,
wildcardDict = self.wildcardSites)
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# Now verify the output datasets
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError, ex:
raise cherrypy.HTTPError(400,
"Bad output dataset name, check the processed dataset.\n %s" %
str(ex))
def handleAssignmentPage(self, **kwargs):
""" handler for the main page """
# handle the checkboxes
teams = []
requestNames = []
for key, value in kwargs.iteritems():
if isinstance(value, types.StringTypes):
kwargs[key] = value.strip()
if key.startswith("Team"):
teams.append(key[4:])
if key.startswith("checkbox"):
requestName = key[8:]
self.validate(requestName)
requestNames.append(key[8:])
for requestName in requestNames:
if kwargs['action'] == 'Reject':
ChangeState.changeRequestStatus(requestName, 'rejected')
else:
assignments = GetRequest.getAssignmentsByName(requestName)
if teams == [] and assignments == []:
raise cherrypy.HTTPError(400, "Must assign to one or more teams")
self.assignWorkload(requestName, kwargs)
for team in teams:
if not team in assignments:
ChangeState.assignRequest(requestName, team)
priority = kwargs.get(requestName+':priority', '')
if priority != '':
Utilities.changePriority(requestName, priority)
participle=kwargs['action']+'ed'
if self.hold and kwargs['action'] == 'Assign':
participle='put into hold state (see <a href="%s">clipboard</a>)' % self.clipboardUrl
requests = [GetRequest.getRequestByName(requestName) for requestName in requestNames]
Clipboard.inject(self.couchUrl, self.clipboardDB, *requests)
for request in requestNames:
ChangeState.changeRequestStatus(requestName, 'ops-hold')
return self.templatepage("Acknowledge", participle=participle, requests=requestNames)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
#Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if type(whiteList) != list:
whiteList = [whiteList]
if type(blackList) != list:
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist = whiteList, siteBlacklist = blackList,
wildcardDict = self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# Now verify the output datasets
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(400,
"Bad output dataset name, check the processed dataset.\n %s" %
str(ex))
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout",None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
useAAA = strToBool(kwargs.get("useSiteListAsLocation", False))
if useAAA:
helper.setLocationDataSourceFlag(flag = useAAA)
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
for site in autoApproveList:
if site.endswith('_MSS'):
raise cherrypy.HTTPError(400, "Auto-approval to MSS endpoint not allowed %s" % autoApproveList)
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority %s" % subscriptionPriority)
custodialType = kwargs.get("CustodialSubType", "Replica")
if custodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % custodialType)
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
if nonCustodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid noncustodial subscription type %s" % nonCustodialType)
helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
custodialSites = custodialList,
nonCustodialSites = nonCustodialList,
autoApproveSites = autoApproveList,
custodialSubType = custodialType,
nonCustodialSubType = nonCustodialType,
priority = subscriptionPriority)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
# set Task properties if they are exist
# TODO: need to define the task format (maybe kwargs["tasks"]?)
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"],
"ProcessingVersion": reqDetails["ProcessingVersion"],
"CustodialSites": custodialList,
"NonCustodialSites": nonCustodialList,
"AutoApproveSubscriptionSites": autoApproveList,
"SubscriptionPriority": subscriptionPriority,
"CustodialSubType": custodialType,
"NonCustodialSubType": nonCustodialType,
"Teams": kwargs["Teams"],
"OutputDatasets": outputDatasets,
"SiteWhitelist": whiteList,
"SiteBlacklist": blackList},
useBody = True)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
# Validate the different parts of the processed dataset
processedDatasetParts = {
"AcquisitionEra": helper.getAcquisitionEra(),
"ProcessingString": helper.getProcessingString(),
"ProcessingVersion": helper.getProcessingVersion(),
}
for field, origValue in processedDatasetParts.iteritems():
if field in kwargs and isinstance(kwargs[field], dict):
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs.get(field, origValue))
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if not isinstance(whiteList, list):
whiteList = [whiteList]
if not isinstance(blackList, list):
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
# Set AcquisitionEra, ProcessingString and ProcessingVersion
# which could be json encoded dicts
if "AcquisitionEra" in kwargs:
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
if "ProcessingString" in kwargs:
helper.setProcessingString(kwargs["ProcessingString"])
if "ProcessingVersion" in kwargs:
helper.setProcessingVersion(kwargs["ProcessingVersion"])
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(400, "Bad output dataset name, check the processed dataset.\n %s" % str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
# FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(
int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)),
)
helper.setupPerformanceMonitoring(
kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None),
)
# Check whether we should check location for the data
helper.setTrustLocationFlag(
inputFlag=strToBool(kwargs.get("TrustSitelists", False)),
pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False)),
)
helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False)))
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
for site in autoApproveList:
if site.endswith("_MSS"):
raise cherrypy.HTTPError(400, "Auto-approval to MSS endpoint not allowed %s" % autoApproveList)
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority %s" % subscriptionPriority)
custodialType = kwargs.get("CustodialSubType", "Replica")
if custodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % custodialType)
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
if nonCustodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid noncustodial subscription type %s" % nonCustodialType)
if "CustodialGroup" in kwargs and not isinstance(kwargs["CustodialGroup"], basestring):
raise cherrypy.HTTPError(400, "Invalid CustodialGroup format %s" % kwargs["CustodialGroup"])
if "NonCustodialGroup" in kwargs and not isinstance(kwargs["NonCustodialGroup"], basestring):
raise cherrypy.HTTPError(400, "Invalid NonCustodialGroup format %s" % kwargs["NonCustodialGroup"])
if "DeleteFromSource" in kwargs and not isinstance(kwargs["DeleteFromSource"], bool):
raise cherrypy.HTTPError(400, "Invalid DeleteFromSource format %s" % kwargs["DeleteFromSource"])
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
custodialGroup=kwargs.get("CustodialGroup", "DataOps"),
nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"),
priority=subscriptionPriority,
deleteFromSource=kwargs.get("DeleteFromSource", False),
)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize)
helper.setMemoryAndCores(kwargs.get("Memory"), kwargs.get("Multicore"))
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request["RequestWorkflow"], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(
request["RequestName"],
"ReqMgr",
"updaterequest",
fields={
"AcquisitionEra": reqDetails["AcquisitionEra"],
"ProcessingVersion": reqDetails["ProcessingVersion"],
"CustodialSites": custodialList,
"NonCustodialSites": nonCustodialList,
"AutoApproveSubscriptionSites": autoApproveList,
"SubscriptionPriority": subscriptionPriority,
"CustodialSubType": custodialType,
"NonCustodialSubType": nonCustodialType,
"CustodialGroup": kwargs.get("CustodialGroup", "DataOps"),
"NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"),
"DeleteFromSource": kwargs.get("DeleteFromSource", False),
"Teams": kwargs["Teams"],
"OutputDatasets": outputDatasets,
"SiteWhitelist": whiteList,
"SiteBlacklist": blackList,
"MergedLFNBase": kwargs["MergedLFNBase"],
"UnmergedLFNBase": kwargs["UnmergedLFNBase"],
"Dashboard": kwargs.get("Dashboard", ""),
"TrustSitelists": kwargs.get("TrustSitelists", False),
"TrustPUSitelists": kwargs.get("TrustPUSitelists", False),
"AllowOpportunistic": kwargs.get("AllowOpportunistic", False),
},
useBody=True,
)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
#Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if type(whiteList) != list:
whiteList = [whiteList]
if type(blackList) != list:
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList,
siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(
400, "White and blacklist the same site is not allowed %s" %
list(res))
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(
400,
"Bad output dataset name, check the processed dataset.\n %s"
% str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
useAAA = strToBool(kwargs.get("useSiteListAsLocation", False))
if useAAA:
helper.setLocationDataSourceFlag(flag=useAAA)
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
for site in autoApproveList:
if site.endswith('_MSS'):
raise cherrypy.HTTPError(
400, "Auto-approval to MSS endpoint not allowed %s" %
autoApproveList)
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(
400, "Invalid subscription priority %s" % subscriptionPriority)
custodialType = kwargs.get("CustodialSubType", "Replica")
if custodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(
400, "Invalid custodial subscription type %s" % custodialType)
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
if nonCustodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(
400,
"Invalid noncustodial subscription type %s" % nonCustodialType)
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
priority=subscriptionPriority)
# Block closing information
blockCloseMaxWaitTime = int(
kwargs.get("BlockCloseMaxWaitTime",
helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(
kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(
kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(
kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
# set Task properties if they are exist
# TODO: need to define the task format (maybe kwargs["tasks"]?)
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'],
self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"],
"ReqMgr",
"updaterequest",
fields={
"AcquisitionEra":
reqDetails["AcquisitionEra"],
"ProcessingVersion":
reqDetails["ProcessingVersion"],
"CustodialSites":
custodialList,
"NonCustodialSites":
nonCustodialList,
"AutoApproveSubscriptionSites":
autoApproveList,
"SubscriptionPriority":
subscriptionPriority,
"CustodialSubType":
custodialType,
"NonCustodialSubType":
nonCustodialType,
"Teams":
kwargs["Teams"],
"OutputDatasets":
outputDatasets,
"SiteWhitelist":
whiteList,
"SiteBlacklist":
blackList
},
useBody=True)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
try:
helper.validateArgumentForAssignment(kwargs)
except WMSpecFactoryException as ex:
raise cherrypy.HTTPError(400, str(ex.message()))
except Exception:
msg = traceback.format_exc()
raise cherrypy.HTTPError(400, "Unhandled error: %s" % msg)
# Validate the different parts of the processed dataset
processedDatasetParts = {"AcquisitionEra": kwargs.get("AcquisitionEra"),
"ProcessingString": kwargs.get("ProcessingString"),
"ProcessingVersion": kwargs.get("ProcessingVersion")
}
for field, values in processedDatasetParts.iteritems():
if field in kwargs and isinstance(kwargs[field], dict):
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs.get(field, values), field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if not isinstance(whiteList, list):
whiteList = [whiteList]
if not isinstance(blackList, list):
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
helper.setAcquisitionEra(kwargs.get("AcquisitionEra", None))
helper.setProcessingString(kwargs.get("ProcessingString", None))
helper.setProcessingVersion(kwargs.get("ProcessingVersion", None))
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(400,
"Bad output dataset name, check the processed dataset.\n %s" %
str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
# FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
helper.setTrustLocationFlag(inputFlag=strToBool(kwargs.get("TrustSitelists", False)),
pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False)))
helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False)))
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
custodialType = kwargs.get("CustodialSubType", "Replica")
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
helper.setSubscriptionInformationWildCards(wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
custodialGroup=kwargs.get("CustodialGroup", "DataOps"),
nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"),
priority=subscriptionPriority,
deleteFromSource=kwargs.get("DeleteFromSource", False))
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setMemory(kwargs.get("Memory"))
helper.setCores(kwargs.get("Multicore"))
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"],
"ProcessingVersion": reqDetails["ProcessingVersion"],
"CustodialSites": custodialList,
"NonCustodialSites": nonCustodialList,
"AutoApproveSubscriptionSites": autoApproveList,
"SubscriptionPriority": subscriptionPriority,
"CustodialSubType": custodialType,
"NonCustodialSubType": nonCustodialType,
"CustodialGroup": kwargs.get("CustodialGroup", "DataOps"),
"NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"),
"DeleteFromSource": kwargs.get("DeleteFromSource", False),
"Teams": kwargs["Teams"],
"OutputDatasets": outputDatasets,
"SiteWhitelist": whiteList,
"SiteBlacklist": blackList,
"MergedLFNBase": kwargs["MergedLFNBase"],
"UnmergedLFNBase": kwargs["UnmergedLFNBase"],
"Dashboard": kwargs.get("Dashboard", ""),
"TrustSitelists": kwargs.get("TrustSitelists", False),
"TrustPUSitelists": kwargs.get("TrustPUSitelists", False),
"AllowOpportunistic": kwargs.get("AllowOpportunistic", False)},
useBody=True)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
# Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites)
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(
int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)),
)
helper.setupPerformanceMonitoring(
int(kwargs.get("maxRSS", 2411724)),
int(kwargs.get("maxVSize", 2411724)),
int(kwargs.get("SoftTimeout", 129600)),
int(kwargs.get("GracePeriod", 300)),
)
# Check whether we should check location for the data
if "useSiteListAsLocation" in kwargs:
helper.setLocationDataSourceFlag()
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority")
subscriptionType = kwargs.get("CustodialSubType", "Move")
if subscriptionType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid custodial subscription type")
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=subscriptionType,
priority=subscriptionPriority,
)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request["RequestWorkflow"], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(
request["RequestName"], "ReqMgr", "updaterequest", fields={"AcquisitionEra": reqDetails["AcquisitionEra"]}
)
