def modifyWorkload(self, requestName, workload,
CMSSWVersion=None, GlobalTag=None,
runWhitelist=None, runBlacklist=None,
blockWhitelist=None, blockBlacklist=None,
ScramArch=None):
""" handles the "Modify" button of the details page """
self.validate(requestName)
helper = WMWorkloadHelper()
helper.load(workload)
schema = helper.data.request.schema
message = ""
if runWhitelist != "" and runWhitelist != None:
l = Utilities.parseRunList(runWhitelist)
helper.setRunWhitelist(l)
schema.RunWhitelist = l
message += 'Changed runWhiteList to %s<br>' % l
if runBlacklist != "" and runBlacklist != None:
l = Utilities.parseRunList(runBlacklist)
helper.setRunBlacklist(l)
schema.RunBlacklist = l
message += 'Changed runBlackList to %s<br>' % l
if blockWhitelist != "" and blockWhitelist != None:
l = Utilities.parseBlockList(blockWhitelist)
helper.setBlockWhitelist(l)
schema.BlockWhitelist = l
message += 'Changed blockWhiteList to %s<br>' % l
if blockBlacklist != "" and blockBlacklist != None:
l = Utilities.parseBlockList(blockBlacklist)
helper.setBlockBlacklist(l)
schema.BlockBlacklist = l
message += 'Changed blockBlackList to %s<br>' % l
Utilities.saveWorkload(helper, workload)
return message + detailsBackLink(requestName)
def handleSplittingPage(self, requestName, splittingTask, splittingAlgo,
**submittedParams):
"""
_handleSplittingPage_
Parse job splitting parameters sent from the splitting parameter update
page. Pull down the request and modify the new spec applying the
updated splitting parameters.
"""
splitParams = {}
if splittingAlgo == "FileBased":
splitParams["files_per_job"] = int(submittedParams["files_per_job"])
elif splittingAlgo == "TwoFileBased":
splitParams["files_per_job"] = int(submittedParams["two_files_per_job"])
elif splittingAlgo == "LumiBased":
splitParams["lumis_per_job"] = int(submittedParams["lumis_per_job"])
if str(submittedParams["halt_job_on_file_boundaries"]) == "True":
splitParams["halt_job_on_file_boundaries"] = True
else:
splitParams["halt_job_on_file_boundaries"] = False
elif splittingAlgo == "EventAwareLumiBased":
splitParams["events_per_job"] = int(submittedParams["avg_events_per_job"])
splitParams["max_events_per_lumi"] = int(submittedParams["max_events_per_lumi"])
if str(submittedParams["halt_job_on_file_boundaries_event_aware"]) == "True":
splitParams["halt_job_on_file_boundaries"] = True
else:
splitParams["halt_job_on_file_boundaries"] = False
elif splittingAlgo == "EventBased":
splitParams["events_per_job"] = int(submittedParams["events_per_job"])
if "events_per_lumi" in submittedParams:
splitParams["events_per_lumi"] = int(submittedParams["events_per_lumi"])
if "lheInputFiles" in submittedParams:
if str(submittedParams["lheInputFiles"]) == "True":
splitParams["lheInputFiles"] = True
else:
splitParams["lheInputFiles"] = False
elif splittingAlgo == "Harvest":
splitParams["periodic_harvest_interval"] = int(submittedParams["periodic_harvest_interval"])
elif 'Merg' in splittingTask:
for field in ['min_merge_size', 'max_merge_size', 'max_merge_events', 'max_wait_time']:
splitParams[field] = int(submittedParams[field])
if "include_parents" in submittedParams.keys():
if str(submittedParams["include_parents"]) == "True":
splitParams["include_parents"] = True
else:
splitParams["include_parents"] = False
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
logging.info("SetSplitting " + requestName + splittingTask + splittingAlgo + str(splitParams))
helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams)
Utilities.saveWorkload(helper, request['RequestWorkflow'])
return "Successfully updated splitting parameters for " + splittingTask \
+ " " + detailsBackLink(requestName)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
for field in ["AcquisitionEra", "ProcessingVersion"]:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
helper.setSiteWildcardsLists(siteWhitelist=whiteList,
siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(
int(kwargs.get("maxRSS", 2411724)),
int(kwargs.get("maxVSize", 2411724)),
int(kwargs.get("SoftTimeout", 167000)),
int(kwargs.get("GracePeriod", 300)))
# Check whether we should check location for the data
if "useSiteListAsLocation" in kwargs:
helper.setLocationDataSourceFlag()
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
if "AutoApprove" in kwargs:
autoApproveList = nonCustodialList
else:
autoApproveList = []
priority = kwargs.get("Priority", "Low")
if priority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority")
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
priority=priority)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'],
self.wmstatWriteURL)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
for field in ["AcquisitionEra", "ProcessingVersion"]:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
helper.setSiteWildcardsLists(siteWhitelist = whiteList, siteBlacklist = blackList,
wildcardDict = self.wildcardSites)
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(int(kwargs.get("maxRSS", 2411724)),
int(kwargs.get("maxVSize", 2411724)),
int(kwargs.get("SoftTimeout", 171600)),
int(kwargs.get("GracePeriod", 300)))
# Check whether we should check location for the data
if "useSiteListAsLocation" in kwargs:
helper.setLocationDataSourceFlag()
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
if "AutoApprove" in kwargs:
autoApproveList = nonCustodialList
else:
autoApproveList = []
priority = kwargs.get("Priority", "Low")
if priority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority")
helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
custodialSites = custodialList,
nonCustodialSites = nonCustodialList,
autoApproveSites = autoApproveList,
priority = priority)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
def modifyWorkload(self, requestName, workload,
CMSSWVersion=None, GlobalTag=None,
runWhitelist=None, runBlacklist=None,
blockWhitelist=None, blockBlacklist=None,
ScramArch=None):
""" handles the "Modify" button of the details page """
self.validate(requestName)
helper = WMWorkloadHelper()
helper.load(workload)
schema = helper.data.request.schema
message = ""
#inputTask = helper.getTask(requestType).data.input.dataset
if GlobalTag or CMSSWVersion:
helper.setCMSSWParams(cmsswVersion=CMSSWVersion, globalTag=GlobalTag)
helper.data.request.schema.CMSSWVersion = CMSSWVersion
helper.data.request.schema.GlobalTag = GlobalTag
message += "CMSSW version %s, GlobalTag %s<br/>" % (CMSSWVersion, GlobalTag)
if runWhitelist != "" and runWhitelist != None:
l = Utilities.parseRunList(runWhitelist)
helper.setRunWhitelist(l)
schema.RunWhitelist = l
message += 'Changed runWhiteList to %s<br>' % l
if runBlacklist != "" and runBlacklist != None:
l = Utilities.parseRunList(runBlacklist)
helper.setRunBlacklist(l)
schema.RunBlacklist = l
message += 'Changed runBlackList to %s<br>' % l
if blockWhitelist != "" and blockWhitelist != None:
l = Utilities.parseBlockList(blockWhitelist)
helper.setBlockWhitelist(l)
schema.BlockWhitelist = l
message += 'Changed blockWhiteList to %s<br>' % l
if blockBlacklist != "" and blockBlacklist != None:
l = Utilities.parseBlockList(blockBlacklist)
helper.setBlockBlacklist(l)
schema.BlockBlacklist = l
message += 'Changed blockBlackList to %s<br>' % l
if ScramArch and ScramArch != schema.ScramArch:
message += "modifyng the Scram Arch to %s" % ScramArch
schema.ScramArch = ScramArch
helper.setCMSSWParams(cmsswVersion=schema.CMSSWVersion, scramArch=ScramArch)
Utilities.saveWorkload(helper, workload)
return message + detailsBackLink(requestName)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
for field in ["AcquisitionEra", "ProcessingVersion"]:
self.validate(kwargs[field], field)
helper.setSiteWhitelist(Utilities.parseSite(kwargs,"SiteWhitelist"))
helper.setSiteBlacklist(Utilities.parseSite(kwargs,"SiteBlacklist"))
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs["MinMergeSize"]),
int(kwargs["MaxMergeSize"]),
int(kwargs["MaxMergeEvents"]))
helper.setupPerformanceMonitoring(int(kwargs["maxRSS"]),
int(kwargs["maxVSize"]))
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'])
def modifyWorkload(self, requestName, workload,
CMSSWVersion=None, GlobalTag=None,
runWhitelist=None, runBlacklist=None,
blockWhitelist=None, blockBlacklist=None,
ScramArch=None):
""" handles the "Modify" button of the details page """
self.validate(requestName)
helper = WMWorkloadHelper()
helper.load(workload)
schema = helper.data.request.schema
message = ""
#inputTask = helper.getTask(requestType).data.input.dataset
if GlobalTag or CMSSWVersion:
helper.setCMSSWParams(cmsswVersion=CMSSWVersion, globalTag=GlobalTag)
helper.data.request.schema.CMSSWVersion = CMSSWVersion
helper.data.request.schema.GlobalTag = GlobalTag
message += "CMSSW version %s, GlobalTag %s<br/>" % (CMSSWVersion, GlobalTag)
if runWhitelist != "" and runWhitelist != None:
l = Utilities.parseRunList(runWhitelist)
helper.setRunWhitelist(l)
schema.RunWhitelist = l
message += 'Changed runWhiteList to %s<br>' % l
if runBlacklist != "" and runBlacklist != None:
l = Utilities.parseRunList(runBlacklist)
helper.setRunBlacklist(l)
schema.RunBlacklist = l
message += 'Changed runBlackList to %s<br>' % l
if blockWhitelist != "" and blockWhitelist != None:
l = Utilities.parseBlockList(blockWhitelist)
helper.setBlockWhitelist(l)
schema.BlockWhitelist = l
message += 'Changed blockWhiteList to %s<br>' % l
if blockBlacklist != "" and blockBlacklist != None:
l = Utilities.parseBlockList(blockBlacklist)
helper.setBlockBlacklist(l)
schema.BlockBlacklist = l
message += 'Changed blockBlackList to %s<br>' % l
if ScramArch and ScramArch != schema.ScramArch:
message += "modifyng the Scram Arch to %s" % ScramArch
schema.ScramArch = ScramArch
helper.setCMSSWParams(cmsswVersion=schema.CMSSWVersion, scramArch=ScramArch)
Utilities.saveWorkload(helper, workload)
return message + detailsBackLink(requestName)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
# Validate the different parts of the processed dataset
processedDatasetParts = {
"AcquisitionEra": helper.getAcquisitionEra(),
"ProcessingString": helper.getProcessingString(),
"ProcessingVersion": helper.getProcessingVersion(),
}
for field, origValue in processedDatasetParts.iteritems():
if field in kwargs and isinstance(kwargs[field], dict):
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs.get(field, origValue))
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if not isinstance(whiteList, list):
whiteList = [whiteList]
if not isinstance(blackList, list):
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
# Set AcquisitionEra, ProcessingString and ProcessingVersion
# which could be json encoded dicts
if "AcquisitionEra" in kwargs:
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
if "ProcessingString" in kwargs:
helper.setProcessingString(kwargs["ProcessingString"])
if "ProcessingVersion" in kwargs:
helper.setProcessingVersion(kwargs["ProcessingVersion"])
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(400, "Bad output dataset name, check the processed dataset.\n %s" % str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
# FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(
int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)),
)
helper.setupPerformanceMonitoring(
kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None),
)
# Check whether we should check location for the data
helper.setTrustLocationFlag(
inputFlag=strToBool(kwargs.get("TrustSitelists", False)),
pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False)),
)
helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False)))
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
for site in autoApproveList:
if site.endswith("_MSS"):
raise cherrypy.HTTPError(400, "Auto-approval to MSS endpoint not allowed %s" % autoApproveList)
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority %s" % subscriptionPriority)
custodialType = kwargs.get("CustodialSubType", "Replica")
if custodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % custodialType)
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
if nonCustodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid noncustodial subscription type %s" % nonCustodialType)
if "CustodialGroup" in kwargs and not isinstance(kwargs["CustodialGroup"], basestring):
raise cherrypy.HTTPError(400, "Invalid CustodialGroup format %s" % kwargs["CustodialGroup"])
if "NonCustodialGroup" in kwargs and not isinstance(kwargs["NonCustodialGroup"], basestring):
raise cherrypy.HTTPError(400, "Invalid NonCustodialGroup format %s" % kwargs["NonCustodialGroup"])
if "DeleteFromSource" in kwargs and not isinstance(kwargs["DeleteFromSource"], bool):
raise cherrypy.HTTPError(400, "Invalid DeleteFromSource format %s" % kwargs["DeleteFromSource"])
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
custodialGroup=kwargs.get("CustodialGroup", "DataOps"),
nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"),
priority=subscriptionPriority,
deleteFromSource=kwargs.get("DeleteFromSource", False),
)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize)
helper.setMemoryAndCores(kwargs.get("Memory"), kwargs.get("Multicore"))
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request["RequestWorkflow"], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(
request["RequestName"],
"ReqMgr",
"updaterequest",
fields={
"AcquisitionEra": reqDetails["AcquisitionEra"],
"ProcessingVersion": reqDetails["ProcessingVersion"],
"CustodialSites": custodialList,
"NonCustodialSites": nonCustodialList,
"AutoApproveSubscriptionSites": autoApproveList,
"SubscriptionPriority": subscriptionPriority,
"CustodialSubType": custodialType,
"NonCustodialSubType": nonCustodialType,
"CustodialGroup": kwargs.get("CustodialGroup", "DataOps"),
"NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"),
"DeleteFromSource": kwargs.get("DeleteFromSource", False),
"Teams": kwargs["Teams"],
"OutputDatasets": outputDatasets,
"SiteWhitelist": whiteList,
"SiteBlacklist": blackList,
"MergedLFNBase": kwargs["MergedLFNBase"],
"UnmergedLFNBase": kwargs["UnmergedLFNBase"],
"Dashboard": kwargs.get("Dashboard", ""),
"TrustSitelists": kwargs.get("TrustSitelists", False),
"TrustPUSitelists": kwargs.get("TrustPUSitelists", False),
"AllowOpportunistic": kwargs.get("AllowOpportunistic", False),
},
useBody=True,
)
# Block closing information
blockCloseMaxWaitTime = int(
kwargs.get("BlockCloseMaxWaitTime",
helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(
kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(
kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(
kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'],
self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"],
"ReqMgr",
"updaterequest",
fields={
"AcquisitionEra":
reqDetails["AcquisitionEra"],
"Teams":
JsonWrapper.JSONEncoder().encode(
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
# Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites)
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(
int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)),
)
helper.setupPerformanceMonitoring(
int(kwargs.get("maxRSS", 2411724)),
int(kwargs.get("maxVSize", 2411724)),
int(kwargs.get("SoftTimeout", 129600)),
int(kwargs.get("GracePeriod", 300)),
)
# Check whether we should check location for the data
if "useSiteListAsLocation" in kwargs:
helper.setLocationDataSourceFlag()
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority")
subscriptionType = kwargs.get("CustodialSubType", "Move")
if subscriptionType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid custodial subscription type")
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=subscriptionType,
priority=subscriptionPriority,
)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request["RequestWorkflow"], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(
request["RequestName"], "ReqMgr", "updaterequest", fields={"AcquisitionEra": reqDetails["AcquisitionEra"]}
)
subscriptionType = kwargs.get("CustodialSubType", "Move")
if subscriptionType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % subscriptionType)
helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
custodialSites = custodialList,
nonCustodialSites = nonCustodialList,
autoApproveSites = autoApproveList,
custodialSubType = subscriptionType,
priority = subscriptionPriority)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"]})
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
try:
helper.validateArgumentForAssignment(kwargs)
except WMSpecFactoryException as ex:
raise cherrypy.HTTPError(400, str(ex.message()))
except Exception:
msg = traceback.format_exc()
raise cherrypy.HTTPError(400, "Unhandled error: %s" % msg)
# Validate the different parts of the processed dataset
processedDatasetParts = {"AcquisitionEra": kwargs.get("AcquisitionEra"),
"ProcessingString": kwargs.get("ProcessingString"),
"ProcessingVersion": kwargs.get("ProcessingVersion")
}
for field, values in processedDatasetParts.iteritems():
if field in kwargs and isinstance(kwargs[field], dict):
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs.get(field, values), field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if not isinstance(whiteList, list):
whiteList = [whiteList]
if not isinstance(blackList, list):
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
helper.setAcquisitionEra(kwargs.get("AcquisitionEra", None))
helper.setProcessingString(kwargs.get("ProcessingString", None))
helper.setProcessingVersion(kwargs.get("ProcessingVersion", None))
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(400,
"Bad output dataset name, check the processed dataset.\n %s" %
str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
# FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
helper.setTrustLocationFlag(inputFlag=strToBool(kwargs.get("TrustSitelists", False)),
pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False)))
helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False)))
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
custodialType = kwargs.get("CustodialSubType", "Replica")
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
helper.setSubscriptionInformationWildCards(wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
custodialGroup=kwargs.get("CustodialGroup", "DataOps"),
nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"),
priority=subscriptionPriority,
deleteFromSource=kwargs.get("DeleteFromSource", False))
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setMemory(kwargs.get("Memory"))
helper.setCores(kwargs.get("Multicore"))
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"],
"ProcessingVersion": reqDetails["ProcessingVersion"],
"CustodialSites": custodialList,
"NonCustodialSites": nonCustodialList,
"AutoApproveSubscriptionSites": autoApproveList,
"SubscriptionPriority": subscriptionPriority,
"CustodialSubType": custodialType,
"NonCustodialSubType": nonCustodialType,
"CustodialGroup": kwargs.get("CustodialGroup", "DataOps"),
"NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"),
"DeleteFromSource": kwargs.get("DeleteFromSource", False),
"Teams": kwargs["Teams"],
"OutputDatasets": outputDatasets,
"SiteWhitelist": whiteList,
"SiteBlacklist": blackList,
"MergedLFNBase": kwargs["MergedLFNBase"],
"UnmergedLFNBase": kwargs["UnmergedLFNBase"],
"Dashboard": kwargs.get("Dashboard", ""),
"TrustSitelists": kwargs.get("TrustSitelists", False),
"TrustPUSitelists": kwargs.get("TrustPUSitelists", False),
"AllowOpportunistic": kwargs.get("AllowOpportunistic", False)},
useBody=True)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
#Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if type(whiteList) != list:
whiteList = [whiteList]
if type(blackList) != list:
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList,
siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(
400, "White and blacklist the same site is not allowed %s" %
list(res))
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(
400,
"Bad output dataset name, check the processed dataset.\n %s"
% str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
useAAA = strToBool(kwargs.get("useSiteListAsLocation", False))
if useAAA:
helper.setLocationDataSourceFlag(flag=useAAA)
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
for site in autoApproveList:
if site.endswith('_MSS'):
raise cherrypy.HTTPError(
400, "Auto-approval to MSS endpoint not allowed %s" %
autoApproveList)
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(
400, "Invalid subscription priority %s" % subscriptionPriority)
custodialType = kwargs.get("CustodialSubType", "Replica")
if custodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(
400, "Invalid custodial subscription type %s" % custodialType)
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
if nonCustodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(
400,
"Invalid noncustodial subscription type %s" % nonCustodialType)
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
priority=subscriptionPriority)
# Block closing information
blockCloseMaxWaitTime = int(
kwargs.get("BlockCloseMaxWaitTime",
helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(
kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(
kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(
kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
# set Task properties if they are exist
# TODO: need to define the task format (maybe kwargs["tasks"]?)
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'],
self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"],
reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"],
"ReqMgr",
"updaterequest",
fields={
"AcquisitionEra":
reqDetails["AcquisitionEra"],
"ProcessingVersion":
reqDetails["ProcessingVersion"],
"CustodialSites":
custodialList,
"NonCustodialSites":
nonCustodialList,
"AutoApproveSubscriptionSites":
autoApproveList,
"SubscriptionPriority":
subscriptionPriority,
"CustodialSubType":
custodialType,
"NonCustodialSubType":
nonCustodialType,
"Teams":
kwargs["Teams"],
"OutputDatasets":
outputDatasets,
"SiteWhitelist":
whiteList,
"SiteBlacklist":
blackList
},
useBody=True)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
#Validate the different parts of the processed dataset
processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"]
if kwargs.get("ProcessingString", None):
processedDatasetParts.append("ProcessingString")
for field in processedDatasetParts:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if type(whiteList) != list:
whiteList = [whiteList]
if type(blackList) != list:
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist = whiteList, siteBlacklist = blackList,
wildcardDict = self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
helper.setProcessingString(kwargs.get("ProcessingString", None))
# Now verify the output datasets
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(400,
"Bad output dataset name, check the processed dataset.\n %s" %
str(ex))
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout",None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
useAAA = strToBool(kwargs.get("useSiteListAsLocation", False))
if useAAA:
helper.setLocationDataSourceFlag(flag = useAAA)
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
for site in autoApproveList:
if site.endswith('_MSS'):
raise cherrypy.HTTPError(400, "Auto-approval to MSS endpoint not allowed %s" % autoApproveList)
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
if subscriptionPriority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority %s" % subscriptionPriority)
custodialType = kwargs.get("CustodialSubType", "Replica")
if custodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % custodialType)
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
if nonCustodialType not in ["Move", "Replica"]:
raise cherrypy.HTTPError(400, "Invalid noncustodial subscription type %s" % nonCustodialType)
helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
custodialSites = custodialList,
nonCustodialSites = nonCustodialList,
autoApproveSites = autoApproveList,
custodialSubType = custodialType,
nonCustodialSubType = nonCustodialType,
priority = subscriptionPriority)
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
# set Task properties if they are exist
# TODO: need to define the task format (maybe kwargs["tasks"]?)
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"],
"ProcessingVersion": reqDetails["ProcessingVersion"],
"CustodialSites": custodialList,
"NonCustodialSites": nonCustodialList,
"AutoApproveSubscriptionSites": autoApproveList,
"SubscriptionPriority": subscriptionPriority,
"CustodialSubType": custodialType,
"NonCustodialSubType": nonCustodialType,
"Teams": kwargs["Teams"],
"OutputDatasets": outputDatasets,
"SiteWhitelist": whiteList,
"SiteBlacklist": blackList},
useBody = True)