def _validateArgument(argument, value, argumentDefinition): validNull = argumentDefinition[argument]["null"] if not validNull and value is None: raise WMSpecFactoryException("Argument %s can't be None" % argument) elif validNull and value is None: return value try: value = argumentDefinition[argument]["type"](value) except Exception: raise WMSpecFactoryException( "Argument: %s: value: %s type is incorrect in schema." % (argument, value)) validateFunction = argumentDefinition[argument]["validate"] if validateFunction is not None: try: if not validateFunction(value): raise WMSpecFactoryException( "Argument %s: value: %s doesn't pass the validation function." % (argument, value)) except Exception as ex: # Some validation functions (e.g. Lexicon) will raise errors instead of returning False logging.error(str(ex)) raise WMSpecFactoryException("Validation failed: %s value: %s" % (argument, value)) return value
def validatePhEDExSubscription(arguments): """ _validatePhEDExSubscription_ Validate all the PhEDEx arguments provided during request creation and assignment. """ for site in arguments.get("AutoApproveSubscriptionSites", []): if site.endswith('_MSS'): raise WMSpecFactoryException( "Auto-approval to MSS endpoint is not allowed: %s" % site) if arguments.get( "SubscriptionPriority", "Low").lower() not in PhEDEx_VALID_SUBSCRIPTION_PRIORITIES: raise WMSpecFactoryException("Invalid subscription priority: %s" % arguments["SubscriptionPriority"]) if arguments.get("CustodialSubType", "Replica") not in ["Move", "Replica"]: raise WMSpecFactoryException( "Invalid custodial subscription type: %s" % arguments["CustodialSubType"]) if arguments.get("NonCustodialSubType", "Replica") not in ["Move", "Replica"]: raise WMSpecFactoryException( "Invalid non custodial subscription type: %s" % arguments["NonCustodialSubType"]) if arguments.get("CustodialSubType") == "Move": _validateMoveSubscription("CustodialSubType", arguments.get('CustodialSites', [])) if arguments.get("NonCustodialSubType") == "Move": _validateMoveSubscription("NonCustodialSubType", arguments.get('NonCustodialSites', [])) return
def _validateArgumentDict(argument, argValue, argumentDefinition): """ Validate arguments that carry a dict value type """ validNull = argumentDefinition["null"] if not validNull and None in argValue.values(): raise WMSpecFactoryException("Argument '%s' cannot be None" % argument) elif all(val is None for val in argValue.values()): return argValue for val in argValue.values(): try: # sigh.. LumiList has a peculiar type validation. # Task/Step is validated later in the schema if argument in ['LumiList', 'Step1', 'Task1']: val = argumentDefinition["type"](argValue) break val = argumentDefinition["type"](val) except Exception: msg = "Argument '%s' with value %r, has an incorrect data type: " % ( argument, val) msg += "%s. It must be %s" % (type(val), argumentDefinition["type"]) raise WMSpecFactoryException(msg) _validateArgFunction(argument, argValue, argumentDefinition["validate"]) return argValue
def _validateArgFunction(argument, value, valFunction): """ Perform the validation function as in the argument definition """ if valFunction: try: if not valFunction(value): raise WMSpecFactoryException("Argument %s, value: %s doesn't pass the validation function." % (argument, value)) except Exception as ex: # Some validation functions (e.g. Lexicon) will raise errors instead of returning False logging.error(str(ex)) raise WMSpecFactoryException("Validation failed: %s value: %s" % (argument, value)) return
def _verifyDBSCall(dbsURL, uri): try: # from WMCore.Services.DBS.DBS3Reader import DBS3Reader # DBS3Reader(dbsURL).dbs.serverinfo() from WMCore.Services.Requests import JSONRequests jsonSender = JSONRequests(dbsURL) result = jsonSender.get("/%s" % uri) if not result[1] == 200: raise WMSpecFactoryException("DBS is not connected: %s : %s" % (dbsURL, str(result))) except: raise WMSpecFactoryException("DBS is not responding: %s" % dbsURL) return result[0]
def validateUnknownArgs(arguments, argumentDefinition): """ Make sure user is sending only arguments that are known by StdBase.getWorkloadCreateArgs, otherwise fail spec creation. It returns an error message if the validation went wrong, otherwise returns None. """ unknownArgs = set(arguments) - set(argumentDefinition.keys()) if unknownArgs: # now onto the exceptions... if arguments.get("RequestType") == "ReReco": unknownArgs = unknownArgs - set( [x for x in unknownArgs if x.startswith("Skim")]) elif arguments.get("RequestType") == "StepChain": unknownArgs = unknownArgs - set( [x for x in unknownArgs if x.startswith("Step")]) elif arguments.get("RequestType") == "TaskChain": unknownArgs = unknownArgs - set( [x for x in unknownArgs if x.startswith("Task")]) elif arguments.get("RequestType") == "Resubmission": # oh well, then we have to skip all possible obscure arguments unknownArgs = unknownArgs - set( [x for x in unknownArgs if x.startswith("Skim")]) unknownArgs = unknownArgs - set( [x for x in unknownArgs if x.startswith("Step")]) unknownArgs = unknownArgs - set( [x for x in unknownArgs if x.startswith("Task")]) if unknownArgs: msg = "There are unknown/unsupported arguments in your request spec: %s" % list( unknownArgs) raise WMSpecFactoryException(msg) return
def validateAutoGenArgument(arguments): autoGenArgs = ["TotalInputEvents", "TotalInputFiles", "TotalInputLumis", "TotalEstimatedJobs"] protectedArgs =set(autoGenArgs).intersection(set(arguments.keys())) if len(protectedArgs) > 0: raise WMSpecFactoryException("Shouldn't set auto generated params %s: remove it" % list(protectedArgs)) return
def _validateArgumentOptions(arguments, argumentDefinition, optionKey): for argument in argumentDefinition: if optionKey == None: optional = True else: optional = argumentDefinition[argument].get(optionKey, True) if not optional and (argument not in arguments): raise WMSpecFactoryException( "Validation failed: %s is mendatory %s" % (argument, argumentDefinition[argument])) #If assign_optional is set to false it need to be assigned later. #TODO this need to be done earlier then this function #elif optionKey == "optional" and not argumentDefinition[argument].get("assign_optional", True): # del arguments[argument] # specific case when user GUI returns empty string for optional arguments elif optional and (argument not in arguments): continue elif optional and (argument in arguments) and (arguments[argument] == ""): del arguments[argument] else: arguments[argument] = _validateArgument(argument, arguments[argument], argumentDefinition) return
def validateInputDatasSetAndParentFlag(arguments): inputdataset = arguments.get("InputDataset", None) if strToBool(arguments.get("IncludeParents", False)): if inputdataset == None: msg = "IncludeParent flag is True but there is no inputdataset" raise WMSpecFactoryException(msg) else: dbsURL = arguments.get("DbsUrl", None) if dbsURL != None: result = _verifyDBSCall(dbsURL, "datasetparents?dataset=%s" % inputdataset) if len(result) == 0: msg = "IncludeParent flag is True but inputdataset %s doesn't have parents" % (inputdataset) raise WMSpecFactoryException(msg) else: _validateInputDataset(arguments) return
def _validateArgument(argument, value, argumentDefinition): """ Validate a single argument against its definition in the spec """ validNull = argumentDefinition["null"] if not validNull and value is None: raise WMSpecFactoryException("Argument %s can't be None" % argument) elif value is None: return value try: value = argumentDefinition["type"](value) except Exception: raise WMSpecFactoryException("Argument: %s: value: %s type is incorrect in schema." % (argument, value)) _validateArgFunction(argument, value, argumentDefinition["validate"]) return value
def _validateMoveSubscription(subType, sites): """ Move subscriptions are only allowed to T0 or T1s, see #7760 """ invalidSites = [site for site in sites if re.match("^T[2-3]", site)] if invalidSites: msg = "Move subscription (%s) not allowed to T2/T3 sites: %s" % (subType, invalidSites) raise WMSpecFactoryException(msg)
def _validateArgFunction(argument, value, valFunction): """ Perform the validation function as in the argument definition """ if valFunction: try: if not valFunction(value): msg = "Argument '%s' with value %r, doesn't pass the validate function." % (argument, value) msg += "\nIt's definition is:\n%s" % inspect.getsource(valFunction) raise WMSpecFactoryException(msg) except WMSpecFactoryException: # just re-raise it to keep the error message clear raise except Exception as ex: # Some validation functions (e.g. Lexicon) will raise errors instead of returning False raise WMSpecFactoryException(str(ex)) return
def makeLumiList(lumiDict): try: if isinstance(lumiDict, basestring): lumiDict = JsonWrapper.loads(lumiDict) ll = LumiList(compactList=lumiDict) return ll.getCompactList() except: raise WMSpecFactoryException("Could not parse LumiList, %s: %s" % (type(lumiDict), lumiDict))
def validatePhEDExSubscription(arguments): """ _validatePhEDExSubscription_ Validate all the PhEDEx arguments provided during request creation and assignment. """ for site in arguments.get("AutoApproveSubscriptionSites", []): if site.endswith('_MSS'): raise WMSpecFactoryException("Auto-approval to MSS endpoint is not allowed: %s" % site) if arguments.get("SubscriptionPriority", "Low") not in ["Low", "Normal", "High"]: raise WMSpecFactoryException("Invalid subscription priority: %s" % arguments["SubscriptionPriority"]) if arguments.get("CustodialSubType", "Replica") not in ["Move", "Replica"]: raise WMSpecFactoryException("Invalid custodial subscription type: %s" % arguments["CustodialSubType"]) if arguments.get("NonCustodialSubType", "Replica") not in ["Move", "Replica"]: raise WMSpecFactoryException("Invalid non custodial subscription type: %s" % arguments["NonCustodialSubType"]) if 'CustodialGroup' in arguments and not isinstance(arguments["CustodialGroup"], basestring): raise WMSpecFactoryException("Invalid custodial PhEDEx group: %s" % arguments["CustodialGroup"]) if 'NonCustodialGroup' in arguments and not isinstance(arguments["NonCustodialGroup"], basestring): raise WMSpecFactoryException("Invalid non custodial PhEDEx group: %s" % arguments["NonCustodialGroup"]) if 'DeleteFromSource' in arguments and not isinstance(arguments["DeleteFromSource"], bool): raise WMSpecFactoryException("Invalid DeleteFromSource type, it must be boolean") return
def _validateInputDataset(arguments): inputdataset = arguments.get("InputDataset", None) dbsURL = arguments.get("DbsUrl", None) if inputdataset != None and dbsURL != None: result = _verifyDBSCall(dbsURL, "datasets?&dataset_access_type=*&dataset=%s" % inputdataset) if len(result) == 0: msg = "Inputdataset %s doesn't exist on %s" % (inputdataset, dbsURL) raise WMSpecFactoryException(msg) return
def validateInputDatasSetAndParentFlag(arguments): inputdataset = arguments.get("InputDataset", None) if strToBool(arguments.get("IncludeParents", False)): if inputdataset == None: msg = "IncludeParent flag is True but there is no inputdataset" raise WMSpecFactoryException(msg) else: dbsURL = arguments.get("DbsUrl", None) if dbsURL != None: #import DBS3Reader here, since Runtime code import this module and worker node doesn't have dbs3 client from WMCore.Services.DBS.DBS3Reader import DBS3Reader result = DBS3Reader(dbsURL).listDatasetParents(inputdataset) if len(result) == 0: msg = "IncludeParent flag is True but inputdataset %s doesn't have parents" % (inputdataset) raise WMSpecFactoryException(msg) else: _validateInputDataset(arguments) return
def makeLumiList(lumiDict): try: if isinstance(lumiDict, (str, bytes)): lumiDict = json.loads(lumiDict) ll = LumiList(compactList=lumiDict) return ll.getCompactList() except: raise WMSpecFactoryException("Could not parse LumiList, %s: %s" % (type(lumiDict), lumiDict))
def safeStr(string): """ _safeStr_ WMCore defined type used to more safely cast simple data types to string """ if not isinstance(string, (tuple, list, set, dict)): return str(string) raise WMSpecFactoryException("We're not supposed to convert %s to string." % string)
def validateInputDatasSetAndParentFlag(arguments): """ Check if the InputDataset value provided corresponds to an actual dataset in DBS. If parent flag is provided, then check whether the input dataset has a parent. the InputDataset existence in DBS and its parent, if needed. """ inputdataset = _getChainKey(arguments, "InputDataset") mcpileup = _getChainKey(arguments, "MCPileup") datapileup = _getChainKey(arguments, "DataPileup") includeParents = _getChainKey(arguments, "IncludeParents") # TODO: this replace can be removed in one year from now, thus March 2022 dbsURL = arguments.get("DbsUrl") if dbsURL: dbsURL = dbsURL.replace("cmsweb.cern.ch", "cmsweb-prod.cern.ch") if includeParents and not inputdataset: msg = "IncludeParents flag is True but InputDataset value has not been provided" raise WMSpecFactoryException(msg) if dbsURL and inputdataset or mcpileup or datapileup: # import DBS3Reader here, since Runtime code import this module and worker # node doesn't have dbs3 client from WMCore.Services.DBS.DBS3Reader import DBS3Reader from WMCore.Services.DBS.DBSErrors import DBSReaderError dbsInst = DBS3Reader(dbsURL) try: _datasetExists(dbsInst, inputdataset) _datasetExists(dbsInst, mcpileup) _datasetExists(dbsInst, datapileup) except DBSReaderError as ex: # we need to Wrap the exception to WMSpecFactoryException to be caught in reqmgr validation raise WMSpecFactoryException(str(ex)) if includeParents: try: result = dbsInst.listDatasetParents(inputdataset) if len(result) == 0: msg = "IncludeParents flag is True but the input dataset %s has no parents" % inputdataset raise DBSReaderError(msg) except DBSReaderError as ex: raise WMSpecFactoryException(str(ex)) return
def _validateArgument(argument, value, argumentDefinition): """ Validate a single argument against its definition in the spec """ validNull = argumentDefinition["null"] if not validNull and value is None: raise WMSpecFactoryException("Argument '%s' cannot be None" % argument) elif value is None: return value try: value = argumentDefinition["type"](value) except Exception: msg = "Argument '%s' with value %r, has an incorrect data type: " % (argument, value) msg += "%s. It must be %s" % (type(value), argumentDefinition["type"]) raise WMSpecFactoryException(msg) _validateArgFunction(argument, value, argumentDefinition["validate"]) return value
def validatePhEDExSubscription(arguments): autoApproveList = arguments.get("AutoApproveSubscriptionSites", []) notAllowedSites = [] for site in autoApproveList: if site.endswith('_MSS'): notAllowedSites.append(site) if len(notAllowedSites) > 0: msg = "Validation failed: Auto-approval to MSS endpoint not allowed: %s" % notAllowedSites raise WMSpecFactoryException(msg) return
def validateSiteLists(arguments): whiteList = arguments.get("SiteWhitelist", []) blackList = arguments.get("SiteBlacklist", []) whiteList = makeList(whiteList) blackList = makeList(blackList) res = (set(whiteList) & set(blackList)) if len(res): msg = "Validation failed: The same site cannot be white and blacklisted: %s" % list( res) raise WMSpecFactoryException(msg) return
def validateSchema(self, schema): """ _validateSchema_ Standard StdBase schema validation, plus verification of the DQMConfigCacheID. """ StdBase.validateSchema(self, schema) if not schema.get("DQMUploadUrl", None): msg = "DQMUploadUrl parameter has not been provided in the request" raise WMSpecFactoryException(message=msg) if not schema.get("DQMConfigCacheID", None): msg = "DQMConfigCacheID parameter has not been provided in the request" raise WMSpecFactoryException(message=msg) couchUrl = schema.get("ConfigCacheUrl", None) self.validateConfigCacheExists(configID=schema["DQMConfigCacheID"], couchURL=couchUrl, couchDBName=schema["CouchDBName"], getOutputModules=False)
def validateSiteLists(arguments): whiteList = arguments.get("SiteWhitelist", []) blackList = arguments.get("SiteBlacklist", []) whiteList = makeList(whiteList) blackList = makeList(blackList) res = (set(whiteList) & set(blackList)) if len(res): msg = "Validation failed: The same site cannot be white and blacklisted: %s" % list(res) raise WMSpecFactoryException(msg) # store the properly formatted values (list instead of string) arguments["SiteWhitelist"] = whiteList arguments["SiteBlacklist"] = blackList return
def _validateArgumentDict(argument, argValue, argumentDefinition): """ Validate arguments that carry a dict value type """ validNull = argumentDefinition["null"] if not validNull and None in argValue.values(): raise WMSpecFactoryException("Argument %s can't be None" % argument) elif all(val is None for val in argValue.values()): return argValue for val in argValue.values(): try: # sigh.. LumiList has a peculiar type validation if argument == 'LumiList': val = argumentDefinition["type"](argValue) break val = argumentDefinition["type"](val) except Exception: raise WMSpecFactoryException("Argument: %s, value: %s type is incorrect in schema." % (argument, val)) _validateArgFunction(argument, argValue, argumentDefinition["validate"]) return argValue
def _validateInputDataset(arguments): inputdataset = arguments.get("InputDataset", None) dbsURL = arguments.get("DbsUrl", None) if inputdataset != None and dbsURL != None: #import DBS3Reader here, since Runtime code import this module and worker node doesn't have dbs3 client from WMCore.Services.DBS.DBS3Reader import DBS3Reader from WMCore.Services.DBS.DBSErrors import DBSReaderError try: DBS3Reader(dbsURL).checkDatasetPath(inputdataset) except DBSReaderError as ex: # we need to Wrap the exception to WMSpecFactoryException to be caught in reqmgr validation raise WMSpecFactoryException(str(ex)) return
def strToBool(string): """ _strToBool_ Convert the string to the matching boolean value: i.e. "True" to python True """ if string == False or string == True: return string # Should we make it more human-friendly (i.e. string in ("Yes", "True", "T")? elif string == "True": return True elif string == "False": return False else: raise WMSpecFactoryException("Can't convert to bool: %s" % string)
def _validateArgumentDict(argument, argValue, argumentDefinition): """ Validate only the basic structure of dict arguments, we anyways don't have the definition of the internal arguments. """ # make sure we're not going to cast a dict to string and let that unnoticed if isinstance(argumentDefinition["type"], type(dict)) and not isinstance(argValue, dict): msg = "Argument '%s' with value %r, has an incorrect data type: " % (argument, argValue) msg += "%s. It must be %s" % (type(argValue), argumentDefinition["type"]) raise WMSpecFactoryException(msg) # still an exception, make sure it has the correct format if argument == "LumiList": argValue = argumentDefinition["type"](argValue) _validateArgFunction(argument, argValue, argumentDefinition["validate"]) return argValue
def makeList(stringList): """ _makeList_ Make a python list out of a comma separated list of strings, throws a WMSpecFactoryException if the input is not well formed. If the stringList is already of type list then it is return untouched """ if isinstance(stringList, list): return stringList if isinstance(stringList, basestring): toks = stringList.lstrip(' [').rstrip(' ]').split(',') if toks == ['']: return [] return [str(tok.strip(' \'"')) for tok in toks] raise WMSpecFactoryException("Can't convert to list %s" % stringList)
def _validateArgumentOptions(arguments, argumentDefinition, optionKey=None): """ Check whether create or assign mandatory parameters were properly set in the request schema. """ for arg, argDef in argumentDefinition.iteritems(): optional = argDef.get(optionKey, True) if not optional and arg not in arguments: msg = "Validation failed: %s parameter is mandatory. Definition: %s" % (arg, argDef) raise WMSpecFactoryException(msg) # specific case when user GUI returns empty string for optional arguments elif arg not in arguments: continue elif isinstance(arguments[arg], dict): arguments[arg] = _validateArgumentDict(arg, arguments[arg], argDef) else: arguments[arg] = _validateArgument(arg, arguments[arg], argDef) return