def modifyWorkload(self, requestName, workload, CMSSWVersion=None, GlobalTag=None, runWhitelist=None, runBlacklist=None, blockWhitelist=None, blockBlacklist=None, ScramArch=None): """ handles the "Modify" button of the details page """ self.validate(requestName) helper = WMWorkloadHelper() helper.load(workload) schema = helper.data.request.schema message = "" if runWhitelist != "" and runWhitelist != None: l = Utilities.parseRunList(runWhitelist) helper.setRunWhitelist(l) schema.RunWhitelist = l message += 'Changed runWhiteList to %s<br>' % l if runBlacklist != "" and runBlacklist != None: l = Utilities.parseRunList(runBlacklist) helper.setRunBlacklist(l) schema.RunBlacklist = l message += 'Changed runBlackList to %s<br>' % l if blockWhitelist != "" and blockWhitelist != None: l = Utilities.parseBlockList(blockWhitelist) helper.setBlockWhitelist(l) schema.BlockWhitelist = l message += 'Changed blockWhiteList to %s<br>' % l if blockBlacklist != "" and blockBlacklist != None: l = Utilities.parseBlockList(blockBlacklist) helper.setBlockBlacklist(l) schema.BlockBlacklist = l message += 'Changed blockBlackList to %s<br>' % l Utilities.saveWorkload(helper, workload) return message + detailsBackLink(requestName)
def handleSplittingPage(self, requestName, splittingTask, splittingAlgo, **submittedParams): """ _handleSplittingPage_ Parse job splitting parameters sent from the splitting parameter update page. Pull down the request and modify the new spec applying the updated splitting parameters. """ splitParams = {} if splittingAlgo == "FileBased": splitParams["files_per_job"] = int(submittedParams["files_per_job"]) elif splittingAlgo == "TwoFileBased": splitParams["files_per_job"] = int(submittedParams["two_files_per_job"]) elif splittingAlgo == "LumiBased": splitParams["lumis_per_job"] = int(submittedParams["lumis_per_job"]) if str(submittedParams["halt_job_on_file_boundaries"]) == "True": splitParams["halt_job_on_file_boundaries"] = True else: splitParams["halt_job_on_file_boundaries"] = False elif splittingAlgo == "EventAwareLumiBased": splitParams["events_per_job"] = int(submittedParams["avg_events_per_job"]) splitParams["max_events_per_lumi"] = int(submittedParams["max_events_per_lumi"]) if str(submittedParams["halt_job_on_file_boundaries_event_aware"]) == "True": splitParams["halt_job_on_file_boundaries"] = True else: splitParams["halt_job_on_file_boundaries"] = False elif splittingAlgo == "EventBased": splitParams["events_per_job"] = int(submittedParams["events_per_job"]) if "events_per_lumi" in submittedParams: splitParams["events_per_lumi"] = int(submittedParams["events_per_lumi"]) if "lheInputFiles" in submittedParams: if str(submittedParams["lheInputFiles"]) == "True": splitParams["lheInputFiles"] = True else: splitParams["lheInputFiles"] = False elif splittingAlgo == "Harvest": splitParams["periodic_harvest_interval"] = int(submittedParams["periodic_harvest_interval"]) elif 'Merg' in splittingTask: for field in ['min_merge_size', 'max_merge_size', 'max_merge_events', 'max_wait_time']: splitParams[field] = int(submittedParams[field]) if "include_parents" in submittedParams.keys(): if str(submittedParams["include_parents"]) == "True": splitParams["include_parents"] = True else: splitParams["include_parents"] = False self.validate(requestName) request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) logging.info("SetSplitting " + requestName + splittingTask + splittingAlgo + str(splitParams)) helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams) Utilities.saveWorkload(helper, request['RequestWorkflow']) return "Successfully updated splitting parameters for " + splittingTask \ + " " + detailsBackLink(requestName)
def assignWorkload(self, requestName, kwargs): """ Make all the necessary changes in the Workload to reflect the new assignment """ request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) for field in ["AcquisitionEra", "ProcessingVersion"]: if type(kwargs[field]) == dict: for value in kwargs[field].values(): self.validate(value, field) else: self.validate(kwargs[field], field) # Set white list and black list whiteList = kwargs.get("SiteWhitelist", []) blackList = kwargs.get("SiteBlacklist", []) helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites) # Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts helper.setProcessingVersion(kwargs["ProcessingVersion"]) helper.setAcquisitionEra(kwargs["AcquisitionEra"]) #FIXME not validated helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"]) helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)), int(kwargs.get("MaxMergeSize", 4294967296)), int(kwargs.get("MaxMergeEvents", 50000))) helper.setupPerformanceMonitoring( int(kwargs.get("maxRSS", 2411724)), int(kwargs.get("maxVSize", 2411724)), int(kwargs.get("SoftTimeout", 167000)), int(kwargs.get("GracePeriod", 300))) # Check whether we should check location for the data if "useSiteListAsLocation" in kwargs: helper.setLocationDataSourceFlag() # Set phedex subscription information custodialList = kwargs.get("CustodialSites", []) nonCustodialList = kwargs.get("NonCustodialSites", []) if "AutoApprove" in kwargs: autoApproveList = nonCustodialList else: autoApproveList = [] priority = kwargs.get("Priority", "Low") if priority not in ["Low", "Normal", "High"]: raise cherrypy.HTTPError(400, "Invalid subscription priority") helper.setSubscriptionInformationWildCards( wildcardDict=self.wildcardSites, custodialSites=custodialList, nonCustodialSites=nonCustodialList, autoApproveSites=autoApproveList, priority=priority) helper.setDashboardActivity(kwargs.get("dashboard", "")) Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
def assignWorkload(self, requestName, kwargs): """ Make all the necessary changes in the Workload to reflect the new assignment """ request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) for field in ["AcquisitionEra", "ProcessingVersion"]: if type(kwargs[field]) == dict: for value in kwargs[field].values(): self.validate(value, field) else: self.validate(kwargs[field], field) # Set white list and black list whiteList = kwargs.get("SiteWhitelist", []) blackList = kwargs.get("SiteBlacklist", []) helper.setSiteWildcardsLists(siteWhitelist = whiteList, siteBlacklist = blackList, wildcardDict = self.wildcardSites) # Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts helper.setProcessingVersion(kwargs["ProcessingVersion"]) helper.setAcquisitionEra(kwargs["AcquisitionEra"]) #FIXME not validated helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"]) helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)), int(kwargs.get("MaxMergeSize", 4294967296)), int(kwargs.get("MaxMergeEvents", 50000))) helper.setupPerformanceMonitoring(int(kwargs.get("maxRSS", 2411724)), int(kwargs.get("maxVSize", 2411724)), int(kwargs.get("SoftTimeout", 171600)), int(kwargs.get("GracePeriod", 300))) # Check whether we should check location for the data if "useSiteListAsLocation" in kwargs: helper.setLocationDataSourceFlag() # Set phedex subscription information custodialList = kwargs.get("CustodialSites", []) nonCustodialList = kwargs.get("NonCustodialSites", []) if "AutoApprove" in kwargs: autoApproveList = nonCustodialList else: autoApproveList = [] priority = kwargs.get("Priority", "Low") if priority not in ["Low", "Normal", "High"]: raise cherrypy.HTTPError(400, "Invalid subscription priority") helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites, custodialSites = custodialList, nonCustodialSites = nonCustodialList, autoApproveSites = autoApproveList, priority = priority) helper.setDashboardActivity(kwargs.get("dashboard", "")) Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
def modifyWorkload(self, requestName, workload, CMSSWVersion=None, GlobalTag=None, runWhitelist=None, runBlacklist=None, blockWhitelist=None, blockBlacklist=None, ScramArch=None): """ handles the "Modify" button of the details page """ self.validate(requestName) helper = WMWorkloadHelper() helper.load(workload) schema = helper.data.request.schema message = "" #inputTask = helper.getTask(requestType).data.input.dataset if GlobalTag or CMSSWVersion: helper.setCMSSWParams(cmsswVersion=CMSSWVersion, globalTag=GlobalTag) helper.data.request.schema.CMSSWVersion = CMSSWVersion helper.data.request.schema.GlobalTag = GlobalTag message += "CMSSW version %s, GlobalTag %s<br/>" % (CMSSWVersion, GlobalTag) if runWhitelist != "" and runWhitelist != None: l = Utilities.parseRunList(runWhitelist) helper.setRunWhitelist(l) schema.RunWhitelist = l message += 'Changed runWhiteList to %s<br>' % l if runBlacklist != "" and runBlacklist != None: l = Utilities.parseRunList(runBlacklist) helper.setRunBlacklist(l) schema.RunBlacklist = l message += 'Changed runBlackList to %s<br>' % l if blockWhitelist != "" and blockWhitelist != None: l = Utilities.parseBlockList(blockWhitelist) helper.setBlockWhitelist(l) schema.BlockWhitelist = l message += 'Changed blockWhiteList to %s<br>' % l if blockBlacklist != "" and blockBlacklist != None: l = Utilities.parseBlockList(blockBlacklist) helper.setBlockBlacklist(l) schema.BlockBlacklist = l message += 'Changed blockBlackList to %s<br>' % l if ScramArch and ScramArch != schema.ScramArch: message += "modifyng the Scram Arch to %s" % ScramArch schema.ScramArch = ScramArch helper.setCMSSWParams(cmsswVersion=schema.CMSSWVersion, scramArch=ScramArch) Utilities.saveWorkload(helper, workload) return message + detailsBackLink(requestName)
def assignWorkload(self, requestName, kwargs): """ Make all the necessary changes in the Workload to reflect the new assignment """ request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) for field in ["AcquisitionEra", "ProcessingVersion"]: self.validate(kwargs[field], field) helper.setSiteWhitelist(Utilities.parseSite(kwargs,"SiteWhitelist")) helper.setSiteBlacklist(Utilities.parseSite(kwargs,"SiteBlacklist")) helper.setProcessingVersion(kwargs["ProcessingVersion"]) helper.setAcquisitionEra(kwargs["AcquisitionEra"]) #FIXME not validated helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"]) helper.setMergeParameters(int(kwargs["MinMergeSize"]), int(kwargs["MaxMergeSize"]), int(kwargs["MaxMergeEvents"])) helper.setupPerformanceMonitoring(int(kwargs["maxRSS"]), int(kwargs["maxVSize"])) helper.setDashboardActivity(kwargs.get("dashboard", "")) Utilities.saveWorkload(helper, request['RequestWorkflow'])
def assignWorkload(self, requestName, kwargs): """ Make all the necessary changes in the Workload to reflect the new assignment """ request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) # Validate the different parts of the processed dataset processedDatasetParts = { "AcquisitionEra": helper.getAcquisitionEra(), "ProcessingString": helper.getProcessingString(), "ProcessingVersion": helper.getProcessingVersion(), } for field, origValue in processedDatasetParts.iteritems(): if field in kwargs and isinstance(kwargs[field], dict): for value in kwargs[field].values(): self.validate(value, field) else: self.validate(kwargs.get(field, origValue)) # Set white list and black list whiteList = kwargs.get("SiteWhitelist", []) blackList = kwargs.get("SiteBlacklist", []) if not isinstance(whiteList, list): whiteList = [whiteList] if not isinstance(blackList, list): blackList = [blackList] helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites) res = set(whiteList) & set(blackList) if len(res): raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res)) # Set AcquisitionEra, ProcessingString and ProcessingVersion # which could be json encoded dicts if "AcquisitionEra" in kwargs: helper.setAcquisitionEra(kwargs["AcquisitionEra"]) if "ProcessingString" in kwargs: helper.setProcessingString(kwargs["ProcessingString"]) if "ProcessingVersion" in kwargs: helper.setProcessingVersion(kwargs["ProcessingVersion"]) # Now verify the output datasets datatier = [] outputDatasets = helper.listOutputDatasets() for dataset in outputDatasets: tokens = dataset.split("/") procds = tokens[2] datatier.append(tokens[3]) try: WMCore.Lexicon.procdataset(procds) except AssertionError as ex: raise cherrypy.HTTPError(400, "Bad output dataset name, check the processed dataset.\n %s" % str(ex)) # Verify whether the output datatiers are available in DBS self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl()) # FIXME not validated helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"]) helper.setMergeParameters( int(kwargs.get("MinMergeSize", 2147483648)), int(kwargs.get("MaxMergeSize", 4294967296)), int(kwargs.get("MaxMergeEvents", 50000)), ) helper.setupPerformanceMonitoring( kwargs.get("MaxRSS", None), kwargs.get("MaxVSize", None), kwargs.get("SoftTimeout", None), kwargs.get("GracePeriod", None), ) # Check whether we should check location for the data helper.setTrustLocationFlag( inputFlag=strToBool(kwargs.get("TrustSitelists", False)), pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False)), ) helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False))) # Set phedex subscription information custodialList = kwargs.get("CustodialSites", []) nonCustodialList = kwargs.get("NonCustodialSites", []) autoApproveList = kwargs.get("AutoApproveSubscriptionSites", []) for site in autoApproveList: if site.endswith("_MSS"): raise cherrypy.HTTPError(400, "Auto-approval to MSS endpoint not allowed %s" % autoApproveList) subscriptionPriority = kwargs.get("SubscriptionPriority", "Low") if subscriptionPriority not in ["Low", "Normal", "High"]: raise cherrypy.HTTPError(400, "Invalid subscription priority %s" % subscriptionPriority) custodialType = kwargs.get("CustodialSubType", "Replica") if custodialType not in ["Move", "Replica"]: raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % custodialType) nonCustodialType = kwargs.get("NonCustodialSubType", "Replica") if nonCustodialType not in ["Move", "Replica"]: raise cherrypy.HTTPError(400, "Invalid noncustodial subscription type %s" % nonCustodialType) if "CustodialGroup" in kwargs and not isinstance(kwargs["CustodialGroup"], basestring): raise cherrypy.HTTPError(400, "Invalid CustodialGroup format %s" % kwargs["CustodialGroup"]) if "NonCustodialGroup" in kwargs and not isinstance(kwargs["NonCustodialGroup"], basestring): raise cherrypy.HTTPError(400, "Invalid NonCustodialGroup format %s" % kwargs["NonCustodialGroup"]) if "DeleteFromSource" in kwargs and not isinstance(kwargs["DeleteFromSource"], bool): raise cherrypy.HTTPError(400, "Invalid DeleteFromSource format %s" % kwargs["DeleteFromSource"]) helper.setSubscriptionInformationWildCards( wildcardDict=self.wildcardSites, custodialSites=custodialList, nonCustodialSites=nonCustodialList, autoApproveSites=autoApproveList, custodialSubType=custodialType, nonCustodialSubType=nonCustodialType, custodialGroup=kwargs.get("CustodialGroup", "DataOps"), nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"), priority=subscriptionPriority, deleteFromSource=kwargs.get("DeleteFromSource", False), ) # Block closing information blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime())) blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles())) blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents())) blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize())) helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize) helper.setMemoryAndCores(kwargs.get("Memory"), kwargs.get("Multicore")) helper.setDashboardActivity(kwargs.get("Dashboard", "")) helper.setTaskProperties(kwargs) Utilities.saveWorkload(helper, request["RequestWorkflow"], self.wmstatWriteURL) # update AcquisitionEra in the Couch document (#4380) # request object returned above from Oracle doesn't have information Couch # database reqDetails = Utilities.requestDetails(request["RequestName"]) couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) couchDb.updateDocument( request["RequestName"], "ReqMgr", "updaterequest", fields={ "AcquisitionEra": reqDetails["AcquisitionEra"], "ProcessingVersion": reqDetails["ProcessingVersion"], "CustodialSites": custodialList, "NonCustodialSites": nonCustodialList, "AutoApproveSubscriptionSites": autoApproveList, "SubscriptionPriority": subscriptionPriority, "CustodialSubType": custodialType, "NonCustodialSubType": nonCustodialType, "CustodialGroup": kwargs.get("CustodialGroup", "DataOps"), "NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"), "DeleteFromSource": kwargs.get("DeleteFromSource", False), "Teams": kwargs["Teams"], "OutputDatasets": outputDatasets, "SiteWhitelist": whiteList, "SiteBlacklist": blackList, "MergedLFNBase": kwargs["MergedLFNBase"], "UnmergedLFNBase": kwargs["UnmergedLFNBase"], "Dashboard": kwargs.get("Dashboard", ""), "TrustSitelists": kwargs.get("TrustSitelists", False), "TrustPUSitelists": kwargs.get("TrustPUSitelists", False), "AllowOpportunistic": kwargs.get("AllowOpportunistic", False), }, useBody=True, )
# Block closing information blockCloseMaxWaitTime = int( kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime())) blockCloseMaxFiles = int( kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles())) blockCloseMaxEvents = int( kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents())) blockCloseMaxSize = int( kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize())) helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize) helper.setDashboardActivity(kwargs.get("dashboard", "")) Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL) # update AcquisitionEra in the Couch document (#4380) # request object returned above from Oracle doesn't have information Couch # database reqDetails = Utilities.requestDetails(request["RequestName"]) couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields={ "AcquisitionEra": reqDetails["AcquisitionEra"], "Teams": JsonWrapper.JSONEncoder().encode(
def assignWorkload(self, requestName, kwargs): """ Make all the necessary changes in the Workload to reflect the new assignment """ request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) # Validate the different parts of the processed dataset processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"] if kwargs.get("ProcessingString", None): processedDatasetParts.append("ProcessingString") for field in processedDatasetParts: if type(kwargs[field]) == dict: for value in kwargs[field].values(): self.validate(value, field) else: self.validate(kwargs[field], field) # Set white list and black list whiteList = kwargs.get("SiteWhitelist", []) blackList = kwargs.get("SiteBlacklist", []) helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites) # Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts helper.setProcessingVersion(kwargs["ProcessingVersion"]) helper.setAcquisitionEra(kwargs["AcquisitionEra"]) helper.setProcessingString(kwargs.get("ProcessingString", None)) # FIXME not validated helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"]) helper.setMergeParameters( int(kwargs.get("MinMergeSize", 2147483648)), int(kwargs.get("MaxMergeSize", 4294967296)), int(kwargs.get("MaxMergeEvents", 50000)), ) helper.setupPerformanceMonitoring( int(kwargs.get("maxRSS", 2411724)), int(kwargs.get("maxVSize", 2411724)), int(kwargs.get("SoftTimeout", 129600)), int(kwargs.get("GracePeriod", 300)), ) # Check whether we should check location for the data if "useSiteListAsLocation" in kwargs: helper.setLocationDataSourceFlag() # Set phedex subscription information custodialList = kwargs.get("CustodialSites", []) nonCustodialList = kwargs.get("NonCustodialSites", []) autoApproveList = kwargs.get("AutoApproveSubscriptionSites", []) subscriptionPriority = kwargs.get("SubscriptionPriority", "Low") if subscriptionPriority not in ["Low", "Normal", "High"]: raise cherrypy.HTTPError(400, "Invalid subscription priority") subscriptionType = kwargs.get("CustodialSubType", "Move") if subscriptionType not in ["Move", "Replica"]: raise cherrypy.HTTPError(400, "Invalid custodial subscription type") helper.setSubscriptionInformationWildCards( wildcardDict=self.wildcardSites, custodialSites=custodialList, nonCustodialSites=nonCustodialList, autoApproveSites=autoApproveList, custodialSubType=subscriptionType, priority=subscriptionPriority, ) # Block closing information blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime())) blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles())) blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents())) blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize())) helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize) helper.setDashboardActivity(kwargs.get("dashboard", "")) Utilities.saveWorkload(helper, request["RequestWorkflow"], self.wmstatWriteURL) # update AcquisitionEra in the Couch document (#4380) # request object returned above from Oracle doesn't have information Couch # database reqDetails = Utilities.requestDetails(request["RequestName"]) couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) couchDb.updateDocument( request["RequestName"], "ReqMgr", "updaterequest", fields={"AcquisitionEra": reqDetails["AcquisitionEra"]} )
subscriptionType = kwargs.get("CustodialSubType", "Move") if subscriptionType not in ["Move", "Replica"]: raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % subscriptionType) helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites, custodialSites = custodialList, nonCustodialSites = nonCustodialList, autoApproveSites = autoApproveList, custodialSubType = subscriptionType, priority = subscriptionPriority) # Block closing information blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime())) blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles())) blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents())) blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize())) helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize) helper.setDashboardActivity(kwargs.get("dashboard", "")) Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL) # update AcquisitionEra in the Couch document (#4380) # request object returned above from Oracle doesn't have information Couch # database reqDetails = Utilities.requestDetails(request["RequestName"]) couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields={"AcquisitionEra": reqDetails["AcquisitionEra"]})
def assignWorkload(self, requestName, kwargs): """ Make all the necessary changes in the Workload to reflect the new assignment """ request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) try: helper.validateArgumentForAssignment(kwargs) except WMSpecFactoryException as ex: raise cherrypy.HTTPError(400, str(ex.message())) except Exception: msg = traceback.format_exc() raise cherrypy.HTTPError(400, "Unhandled error: %s" % msg) # Validate the different parts of the processed dataset processedDatasetParts = {"AcquisitionEra": kwargs.get("AcquisitionEra"), "ProcessingString": kwargs.get("ProcessingString"), "ProcessingVersion": kwargs.get("ProcessingVersion") } for field, values in processedDatasetParts.iteritems(): if field in kwargs and isinstance(kwargs[field], dict): for value in kwargs[field].values(): self.validate(value, field) else: self.validate(kwargs.get(field, values), field) # Set white list and black list whiteList = kwargs.get("SiteWhitelist", []) blackList = kwargs.get("SiteBlacklist", []) if not isinstance(whiteList, list): whiteList = [whiteList] if not isinstance(blackList, list): blackList = [blackList] helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites) res = set(whiteList) & set(blackList) if len(res): raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res)) helper.setAcquisitionEra(kwargs.get("AcquisitionEra", None)) helper.setProcessingString(kwargs.get("ProcessingString", None)) helper.setProcessingVersion(kwargs.get("ProcessingVersion", None)) # Now verify the output datasets datatier = [] outputDatasets = helper.listOutputDatasets() for dataset in outputDatasets: tokens = dataset.split("/") procds = tokens[2] datatier.append(tokens[3]) try: WMCore.Lexicon.procdataset(procds) except AssertionError as ex: raise cherrypy.HTTPError(400, "Bad output dataset name, check the processed dataset.\n %s" % str(ex)) # Verify whether the output datatiers are available in DBS self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl()) # FIXME not validated helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"]) helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)), int(kwargs.get("MaxMergeSize", 4294967296)), int(kwargs.get("MaxMergeEvents", 50000))) helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None), kwargs.get("MaxVSize", None), kwargs.get("SoftTimeout", None), kwargs.get("GracePeriod", None)) # Check whether we should check location for the data helper.setTrustLocationFlag(inputFlag=strToBool(kwargs.get("TrustSitelists", False)), pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False))) helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False))) # Set phedex subscription information custodialList = kwargs.get("CustodialSites", []) nonCustodialList = kwargs.get("NonCustodialSites", []) autoApproveList = kwargs.get("AutoApproveSubscriptionSites", []) subscriptionPriority = kwargs.get("SubscriptionPriority", "Low") custodialType = kwargs.get("CustodialSubType", "Replica") nonCustodialType = kwargs.get("NonCustodialSubType", "Replica") helper.setSubscriptionInformationWildCards(wildcardDict=self.wildcardSites, custodialSites=custodialList, nonCustodialSites=nonCustodialList, autoApproveSites=autoApproveList, custodialSubType=custodialType, nonCustodialSubType=nonCustodialType, custodialGroup=kwargs.get("CustodialGroup", "DataOps"), nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"), priority=subscriptionPriority, deleteFromSource=kwargs.get("DeleteFromSource", False)) # Block closing information blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime())) blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles())) blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents())) blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize())) helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize) helper.setMemory(kwargs.get("Memory")) helper.setCores(kwargs.get("Multicore")) helper.setDashboardActivity(kwargs.get("Dashboard", "")) helper.setTaskProperties(kwargs) Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL) # update AcquisitionEra in the Couch document (#4380) # request object returned above from Oracle doesn't have information Couch # database reqDetails = Utilities.requestDetails(request["RequestName"]) couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields={"AcquisitionEra": reqDetails["AcquisitionEra"], "ProcessingVersion": reqDetails["ProcessingVersion"], "CustodialSites": custodialList, "NonCustodialSites": nonCustodialList, "AutoApproveSubscriptionSites": autoApproveList, "SubscriptionPriority": subscriptionPriority, "CustodialSubType": custodialType, "NonCustodialSubType": nonCustodialType, "CustodialGroup": kwargs.get("CustodialGroup", "DataOps"), "NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"), "DeleteFromSource": kwargs.get("DeleteFromSource", False), "Teams": kwargs["Teams"], "OutputDatasets": outputDatasets, "SiteWhitelist": whiteList, "SiteBlacklist": blackList, "MergedLFNBase": kwargs["MergedLFNBase"], "UnmergedLFNBase": kwargs["UnmergedLFNBase"], "Dashboard": kwargs.get("Dashboard", ""), "TrustSitelists": kwargs.get("TrustSitelists", False), "TrustPUSitelists": kwargs.get("TrustPUSitelists", False), "AllowOpportunistic": kwargs.get("AllowOpportunistic", False)}, useBody=True)
def assignWorkload(self, requestName, kwargs): """ Make all the necessary changes in the Workload to reflect the new assignment """ request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) #Validate the different parts of the processed dataset processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"] if kwargs.get("ProcessingString", None): processedDatasetParts.append("ProcessingString") for field in processedDatasetParts: if type(kwargs[field]) == dict: for value in kwargs[field].values(): self.validate(value, field) else: self.validate(kwargs[field], field) # Set white list and black list whiteList = kwargs.get("SiteWhitelist", []) blackList = kwargs.get("SiteBlacklist", []) if type(whiteList) != list: whiteList = [whiteList] if type(blackList) != list: blackList = [blackList] helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites) res = set(whiteList) & set(blackList) if len(res): raise cherrypy.HTTPError( 400, "White and blacklist the same site is not allowed %s" % list(res)) # Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts helper.setProcessingVersion(kwargs["ProcessingVersion"]) helper.setAcquisitionEra(kwargs["AcquisitionEra"]) helper.setProcessingString(kwargs.get("ProcessingString", None)) # Now verify the output datasets datatier = [] outputDatasets = helper.listOutputDatasets() for dataset in outputDatasets: tokens = dataset.split("/") procds = tokens[2] datatier.append(tokens[3]) try: WMCore.Lexicon.procdataset(procds) except AssertionError as ex: raise cherrypy.HTTPError( 400, "Bad output dataset name, check the processed dataset.\n %s" % str(ex)) # Verify whether the output datatiers are available in DBS self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl()) #FIXME not validated helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"]) helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)), int(kwargs.get("MaxMergeSize", 4294967296)), int(kwargs.get("MaxMergeEvents", 50000))) helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None), kwargs.get("MaxVSize", None), kwargs.get("SoftTimeout", None), kwargs.get("GracePeriod", None)) # Check whether we should check location for the data useAAA = strToBool(kwargs.get("useSiteListAsLocation", False)) if useAAA: helper.setLocationDataSourceFlag(flag=useAAA) # Set phedex subscription information custodialList = kwargs.get("CustodialSites", []) nonCustodialList = kwargs.get("NonCustodialSites", []) autoApproveList = kwargs.get("AutoApproveSubscriptionSites", []) for site in autoApproveList: if site.endswith('_MSS'): raise cherrypy.HTTPError( 400, "Auto-approval to MSS endpoint not allowed %s" % autoApproveList) subscriptionPriority = kwargs.get("SubscriptionPriority", "Low") if subscriptionPriority not in ["Low", "Normal", "High"]: raise cherrypy.HTTPError( 400, "Invalid subscription priority %s" % subscriptionPriority) custodialType = kwargs.get("CustodialSubType", "Replica") if custodialType not in ["Move", "Replica"]: raise cherrypy.HTTPError( 400, "Invalid custodial subscription type %s" % custodialType) nonCustodialType = kwargs.get("NonCustodialSubType", "Replica") if nonCustodialType not in ["Move", "Replica"]: raise cherrypy.HTTPError( 400, "Invalid noncustodial subscription type %s" % nonCustodialType) helper.setSubscriptionInformationWildCards( wildcardDict=self.wildcardSites, custodialSites=custodialList, nonCustodialSites=nonCustodialList, autoApproveSites=autoApproveList, custodialSubType=custodialType, nonCustodialSubType=nonCustodialType, priority=subscriptionPriority) # Block closing information blockCloseMaxWaitTime = int( kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime())) blockCloseMaxFiles = int( kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles())) blockCloseMaxEvents = int( kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents())) blockCloseMaxSize = int( kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize())) helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize) helper.setDashboardActivity(kwargs.get("Dashboard", "")) # set Task properties if they are exist # TODO: need to define the task format (maybe kwargs["tasks"]?) helper.setTaskProperties(kwargs) Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL) # update AcquisitionEra in the Couch document (#4380) # request object returned above from Oracle doesn't have information Couch # database reqDetails = Utilities.requestDetails(request["RequestName"]) couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields={ "AcquisitionEra": reqDetails["AcquisitionEra"], "ProcessingVersion": reqDetails["ProcessingVersion"], "CustodialSites": custodialList, "NonCustodialSites": nonCustodialList, "AutoApproveSubscriptionSites": autoApproveList, "SubscriptionPriority": subscriptionPriority, "CustodialSubType": custodialType, "NonCustodialSubType": nonCustodialType, "Teams": kwargs["Teams"], "OutputDatasets": outputDatasets, "SiteWhitelist": whiteList, "SiteBlacklist": blackList }, useBody=True)
def assignWorkload(self, requestName, kwargs): """ Make all the necessary changes in the Workload to reflect the new assignment """ request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) #Validate the different parts of the processed dataset processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"] if kwargs.get("ProcessingString", None): processedDatasetParts.append("ProcessingString") for field in processedDatasetParts: if type(kwargs[field]) == dict: for value in kwargs[field].values(): self.validate(value, field) else: self.validate(kwargs[field], field) # Set white list and black list whiteList = kwargs.get("SiteWhitelist", []) blackList = kwargs.get("SiteBlacklist", []) if type(whiteList) != list: whiteList = [whiteList] if type(blackList) != list: blackList = [blackList] helper.setSiteWildcardsLists(siteWhitelist = whiteList, siteBlacklist = blackList, wildcardDict = self.wildcardSites) res = set(whiteList) & set(blackList) if len(res): raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res)) # Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts helper.setProcessingVersion(kwargs["ProcessingVersion"]) helper.setAcquisitionEra(kwargs["AcquisitionEra"]) helper.setProcessingString(kwargs.get("ProcessingString", None)) # Now verify the output datasets outputDatasets = helper.listOutputDatasets() for dataset in outputDatasets: tokens = dataset.split("/") procds = tokens[2] try: WMCore.Lexicon.procdataset(procds) except AssertionError as ex: raise cherrypy.HTTPError(400, "Bad output dataset name, check the processed dataset.\n %s" % str(ex)) #FIXME not validated helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"]) helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)), int(kwargs.get("MaxMergeSize", 4294967296)), int(kwargs.get("MaxMergeEvents", 50000))) helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None), kwargs.get("MaxVSize", None), kwargs.get("SoftTimeout",None), kwargs.get("GracePeriod", None)) # Check whether we should check location for the data useAAA = strToBool(kwargs.get("useSiteListAsLocation", False)) if useAAA: helper.setLocationDataSourceFlag(flag = useAAA) # Set phedex subscription information custodialList = kwargs.get("CustodialSites", []) nonCustodialList = kwargs.get("NonCustodialSites", []) autoApproveList = kwargs.get("AutoApproveSubscriptionSites", []) for site in autoApproveList: if site.endswith('_MSS'): raise cherrypy.HTTPError(400, "Auto-approval to MSS endpoint not allowed %s" % autoApproveList) subscriptionPriority = kwargs.get("SubscriptionPriority", "Low") if subscriptionPriority not in ["Low", "Normal", "High"]: raise cherrypy.HTTPError(400, "Invalid subscription priority %s" % subscriptionPriority) custodialType = kwargs.get("CustodialSubType", "Replica") if custodialType not in ["Move", "Replica"]: raise cherrypy.HTTPError(400, "Invalid custodial subscription type %s" % custodialType) nonCustodialType = kwargs.get("NonCustodialSubType", "Replica") if nonCustodialType not in ["Move", "Replica"]: raise cherrypy.HTTPError(400, "Invalid noncustodial subscription type %s" % nonCustodialType) helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites, custodialSites = custodialList, nonCustodialSites = nonCustodialList, autoApproveSites = autoApproveList, custodialSubType = custodialType, nonCustodialSubType = nonCustodialType, priority = subscriptionPriority) # Block closing information blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime())) blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles())) blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents())) blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize())) helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize) helper.setDashboardActivity(kwargs.get("Dashboard", "")) # set Task properties if they are exist # TODO: need to define the task format (maybe kwargs["tasks"]?) helper.setTaskProperties(kwargs) Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL) # update AcquisitionEra in the Couch document (#4380) # request object returned above from Oracle doesn't have information Couch # database reqDetails = Utilities.requestDetails(request["RequestName"]) couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields={"AcquisitionEra": reqDetails["AcquisitionEra"], "ProcessingVersion": reqDetails["ProcessingVersion"], "CustodialSites": custodialList, "NonCustodialSites": nonCustodialList, "AutoApproveSubscriptionSites": autoApproveList, "SubscriptionPriority": subscriptionPriority, "CustodialSubType": custodialType, "NonCustodialSubType": nonCustodialType, "Teams": kwargs["Teams"], "OutputDatasets": outputDatasets, "SiteWhitelist": whiteList, "SiteBlacklist": blackList}, useBody = True)