def _validateArgument(argument, value, argumentDefinition):
validNull = argumentDefinition[argument]["null"]
if not validNull and value is None:
raise WMSpecFactoryException("Argument %s can't be None" % argument)
elif validNull and value is None:
return value
try:
value = argumentDefinition[argument]["type"](value)
except Exception:
raise WMSpecFactoryException(
"Argument: %s: value: %s type is incorrect in schema." %
(argument, value))
validateFunction = argumentDefinition[argument]["validate"]
if validateFunction is not None:
try:
if not validateFunction(value):
raise WMSpecFactoryException(
"Argument %s: value: %s doesn't pass the validation function."
% (argument, value))
except Exception as ex:
# Some validation functions (e.g. Lexicon) will raise errors instead of returning False
logging.error(str(ex))
raise WMSpecFactoryException("Validation failed: %s value: %s" %
(argument, value))
return value
def validatePhEDExSubscription(arguments):
"""
_validatePhEDExSubscription_
Validate all the PhEDEx arguments provided during request
creation and assignment.
"""
for site in arguments.get("AutoApproveSubscriptionSites", []):
if site.endswith('_MSS'):
raise WMSpecFactoryException(
"Auto-approval to MSS endpoint is not allowed: %s" % site)
if arguments.get(
"SubscriptionPriority",
"Low").lower() not in PhEDEx_VALID_SUBSCRIPTION_PRIORITIES:
raise WMSpecFactoryException("Invalid subscription priority: %s" %
arguments["SubscriptionPriority"])
if arguments.get("CustodialSubType", "Replica") not in ["Move", "Replica"]:
raise WMSpecFactoryException(
"Invalid custodial subscription type: %s" %
arguments["CustodialSubType"])
if arguments.get("NonCustodialSubType",
"Replica") not in ["Move", "Replica"]:
raise WMSpecFactoryException(
"Invalid non custodial subscription type: %s" %
arguments["NonCustodialSubType"])
if arguments.get("CustodialSubType") == "Move":
_validateMoveSubscription("CustodialSubType",
arguments.get('CustodialSites', []))
if arguments.get("NonCustodialSubType") == "Move":
_validateMoveSubscription("NonCustodialSubType",
arguments.get('NonCustodialSites', []))
return
def _validateArgumentDict(argument, argValue, argumentDefinition):
"""
Validate arguments that carry a dict value type
"""
validNull = argumentDefinition["null"]
if not validNull and None in argValue.values():
raise WMSpecFactoryException("Argument '%s' cannot be None" % argument)
elif all(val is None for val in argValue.values()):
return argValue
for val in argValue.values():
try:
# sigh.. LumiList has a peculiar type validation.
# Task/Step is validated later in the schema
if argument in ['LumiList', 'Step1', 'Task1']:
val = argumentDefinition["type"](argValue)
break
val = argumentDefinition["type"](val)
except Exception:
msg = "Argument '%s' with value %r, has an incorrect data type: " % (
argument, val)
msg += "%s. It must be %s" % (type(val),
argumentDefinition["type"])
raise WMSpecFactoryException(msg)
_validateArgFunction(argument, argValue, argumentDefinition["validate"])
return argValue
def _validateArgFunction(argument, value, valFunction):
"""
Perform the validation function as in the argument definition
"""
if valFunction:
try:
if not valFunction(value):
raise WMSpecFactoryException("Argument %s, value: %s doesn't pass the validation function." % (argument, value))
except Exception as ex:
# Some validation functions (e.g. Lexicon) will raise errors instead of returning False
logging.error(str(ex))
raise WMSpecFactoryException("Validation failed: %s value: %s" % (argument, value))
return
def _verifyDBSCall(dbsURL, uri):
try:
# from WMCore.Services.DBS.DBS3Reader import DBS3Reader
# DBS3Reader(dbsURL).dbs.serverinfo()
from WMCore.Services.Requests import JSONRequests
jsonSender = JSONRequests(dbsURL)
result = jsonSender.get("/%s" % uri)
if not result[1] == 200:
raise WMSpecFactoryException("DBS is not connected: %s : %s" % (dbsURL, str(result)))
except:
raise WMSpecFactoryException("DBS is not responding: %s" % dbsURL)
return result[0]
def validateUnknownArgs(arguments, argumentDefinition):
"""
Make sure user is sending only arguments that are known by
StdBase.getWorkloadCreateArgs, otherwise fail spec creation.
It returns an error message if the validation went wrong,
otherwise returns None.
"""
unknownArgs = set(arguments) - set(argumentDefinition.keys())
if unknownArgs:
# now onto the exceptions...
if arguments.get("RequestType") == "ReReco":
unknownArgs = unknownArgs - set(
[x for x in unknownArgs if x.startswith("Skim")])
elif arguments.get("RequestType") == "StepChain":
unknownArgs = unknownArgs - set(
[x for x in unknownArgs if x.startswith("Step")])
elif arguments.get("RequestType") == "TaskChain":
unknownArgs = unknownArgs - set(
[x for x in unknownArgs if x.startswith("Task")])
elif arguments.get("RequestType") == "Resubmission":
# oh well, then we have to skip all possible obscure arguments
unknownArgs = unknownArgs - set(
[x for x in unknownArgs if x.startswith("Skim")])
unknownArgs = unknownArgs - set(
[x for x in unknownArgs if x.startswith("Step")])
unknownArgs = unknownArgs - set(
[x for x in unknownArgs if x.startswith("Task")])
if unknownArgs:
msg = "There are unknown/unsupported arguments in your request spec: %s" % list(
unknownArgs)
raise WMSpecFactoryException(msg)
return
def validateAutoGenArgument(arguments):
autoGenArgs = ["TotalInputEvents", "TotalInputFiles", "TotalInputLumis", "TotalEstimatedJobs"]
protectedArgs =set(autoGenArgs).intersection(set(arguments.keys()))
if len(protectedArgs) > 0:
raise WMSpecFactoryException("Shouldn't set auto generated params %s: remove it" % list(protectedArgs))
return
def _validateArgumentOptions(arguments, argumentDefinition, optionKey):
for argument in argumentDefinition:
if optionKey == None:
optional = True
else:
optional = argumentDefinition[argument].get(optionKey, True)
if not optional and (argument not in arguments):
raise WMSpecFactoryException(
"Validation failed: %s is mendatory %s" %
(argument, argumentDefinition[argument]))
#If assign_optional is set to false it need to be assigned later.
#TODO this need to be done earlier then this function
#elif optionKey == "optional" and not argumentDefinition[argument].get("assign_optional", True):
# del arguments[argument]
# specific case when user GUI returns empty string for optional arguments
elif optional and (argument not in arguments):
continue
elif optional and (argument in arguments) and (arguments[argument]
== ""):
del arguments[argument]
else:
arguments[argument] = _validateArgument(argument,
arguments[argument],
argumentDefinition)
return
def validateInputDatasSetAndParentFlag(arguments):
inputdataset = arguments.get("InputDataset", None)
if strToBool(arguments.get("IncludeParents", False)):
if inputdataset == None:
msg = "IncludeParent flag is True but there is no inputdataset"
raise WMSpecFactoryException(msg)
else:
dbsURL = arguments.get("DbsUrl", None)
if dbsURL != None:
result = _verifyDBSCall(dbsURL, "datasetparents?dataset=%s" % inputdataset)
if len(result) == 0:
msg = "IncludeParent flag is True but inputdataset %s doesn't have parents" % (inputdataset)
raise WMSpecFactoryException(msg)
else:
_validateInputDataset(arguments)
return
def _validateArgument(argument, value, argumentDefinition):
"""
Validate a single argument against its definition in the spec
"""
validNull = argumentDefinition["null"]
if not validNull and value is None:
raise WMSpecFactoryException("Argument %s can't be None" % argument)
elif value is None:
return value
try:
value = argumentDefinition["type"](value)
except Exception:
raise WMSpecFactoryException("Argument: %s: value: %s type is incorrect in schema." % (argument, value))
_validateArgFunction(argument, value, argumentDefinition["validate"])
return value
def _validateMoveSubscription(subType, sites):
"""
Move subscriptions are only allowed to T0 or T1s, see #7760
"""
invalidSites = [site for site in sites if re.match("^T[2-3]", site)]
if invalidSites:
msg = "Move subscription (%s) not allowed to T2/T3 sites: %s" % (subType, invalidSites)
raise WMSpecFactoryException(msg)
def _validateArgFunction(argument, value, valFunction):
"""
Perform the validation function as in the argument definition
"""
if valFunction:
try:
if not valFunction(value):
msg = "Argument '%s' with value %r, doesn't pass the validate function." % (argument, value)
msg += "\nIt's definition is:\n%s" % inspect.getsource(valFunction)
raise WMSpecFactoryException(msg)
except WMSpecFactoryException:
# just re-raise it to keep the error message clear
raise
except Exception as ex:
# Some validation functions (e.g. Lexicon) will raise errors instead of returning False
raise WMSpecFactoryException(str(ex))
return
def makeLumiList(lumiDict):
try:
if isinstance(lumiDict, basestring):
lumiDict = JsonWrapper.loads(lumiDict)
ll = LumiList(compactList=lumiDict)
return ll.getCompactList()
except:
raise WMSpecFactoryException("Could not parse LumiList, %s: %s" % (type(lumiDict), lumiDict))
def validatePhEDExSubscription(arguments):
"""
_validatePhEDExSubscription_
Validate all the PhEDEx arguments provided during request
creation and assignment.
"""
for site in arguments.get("AutoApproveSubscriptionSites", []):
if site.endswith('_MSS'):
raise WMSpecFactoryException("Auto-approval to MSS endpoint is not allowed: %s" % site)
if arguments.get("SubscriptionPriority", "Low") not in ["Low", "Normal", "High"]:
raise WMSpecFactoryException("Invalid subscription priority: %s" % arguments["SubscriptionPriority"])
if arguments.get("CustodialSubType", "Replica") not in ["Move", "Replica"]:
raise WMSpecFactoryException("Invalid custodial subscription type: %s" % arguments["CustodialSubType"])
if arguments.get("NonCustodialSubType", "Replica") not in ["Move", "Replica"]:
raise WMSpecFactoryException("Invalid non custodial subscription type: %s" % arguments["NonCustodialSubType"])
if 'CustodialGroup' in arguments and not isinstance(arguments["CustodialGroup"], basestring):
raise WMSpecFactoryException("Invalid custodial PhEDEx group: %s" % arguments["CustodialGroup"])
if 'NonCustodialGroup' in arguments and not isinstance(arguments["NonCustodialGroup"], basestring):
raise WMSpecFactoryException("Invalid non custodial PhEDEx group: %s" % arguments["NonCustodialGroup"])
if 'DeleteFromSource' in arguments and not isinstance(arguments["DeleteFromSource"], bool):
raise WMSpecFactoryException("Invalid DeleteFromSource type, it must be boolean")
return
def _validateInputDataset(arguments):
inputdataset = arguments.get("InputDataset", None)
dbsURL = arguments.get("DbsUrl", None)
if inputdataset != None and dbsURL != None:
result = _verifyDBSCall(dbsURL, "datasets?&dataset_access_type=*&dataset=%s" % inputdataset)
if len(result) == 0:
msg = "Inputdataset %s doesn't exist on %s" % (inputdataset, dbsURL)
raise WMSpecFactoryException(msg)
return
def validateInputDatasSetAndParentFlag(arguments):
inputdataset = arguments.get("InputDataset", None)
if strToBool(arguments.get("IncludeParents", False)):
if inputdataset == None:
msg = "IncludeParent flag is True but there is no inputdataset"
raise WMSpecFactoryException(msg)
else:
dbsURL = arguments.get("DbsUrl", None)
if dbsURL != None:
#import DBS3Reader here, since Runtime code import this module and worker node doesn't have dbs3 client
from WMCore.Services.DBS.DBS3Reader import DBS3Reader
result = DBS3Reader(dbsURL).listDatasetParents(inputdataset)
if len(result) == 0:
msg = "IncludeParent flag is True but inputdataset %s doesn't have parents" % (inputdataset)
raise WMSpecFactoryException(msg)
else:
_validateInputDataset(arguments)
return
def makeLumiList(lumiDict):
try:
if isinstance(lumiDict, (str, bytes)):
lumiDict = json.loads(lumiDict)
ll = LumiList(compactList=lumiDict)
return ll.getCompactList()
except:
raise WMSpecFactoryException("Could not parse LumiList, %s: %s" %
(type(lumiDict), lumiDict))
def safeStr(string):
"""
_safeStr_
WMCore defined type used to more safely cast simple data types to string
"""
if not isinstance(string, (tuple, list, set, dict)):
return str(string)
raise WMSpecFactoryException("We're not supposed to convert %s to string." % string)
def validateInputDatasSetAndParentFlag(arguments):
"""
Check if the InputDataset value provided corresponds to an actual dataset in DBS.
If parent flag is provided, then check whether the input dataset has a parent.
the InputDataset existence in DBS and its parent, if needed.
"""
inputdataset = _getChainKey(arguments, "InputDataset")
mcpileup = _getChainKey(arguments, "MCPileup")
datapileup = _getChainKey(arguments, "DataPileup")
includeParents = _getChainKey(arguments, "IncludeParents")
# TODO: this replace can be removed in one year from now, thus March 2022
dbsURL = arguments.get("DbsUrl")
if dbsURL:
dbsURL = dbsURL.replace("cmsweb.cern.ch", "cmsweb-prod.cern.ch")
if includeParents and not inputdataset:
msg = "IncludeParents flag is True but InputDataset value has not been provided"
raise WMSpecFactoryException(msg)
if dbsURL and inputdataset or mcpileup or datapileup:
# import DBS3Reader here, since Runtime code import this module and worker
# node doesn't have dbs3 client
from WMCore.Services.DBS.DBS3Reader import DBS3Reader
from WMCore.Services.DBS.DBSErrors import DBSReaderError
dbsInst = DBS3Reader(dbsURL)
try:
_datasetExists(dbsInst, inputdataset)
_datasetExists(dbsInst, mcpileup)
_datasetExists(dbsInst, datapileup)
except DBSReaderError as ex:
# we need to Wrap the exception to WMSpecFactoryException to be caught in reqmgr validation
raise WMSpecFactoryException(str(ex))
if includeParents:
try:
result = dbsInst.listDatasetParents(inputdataset)
if len(result) == 0:
msg = "IncludeParents flag is True but the input dataset %s has no parents" % inputdataset
raise DBSReaderError(msg)
except DBSReaderError as ex:
raise WMSpecFactoryException(str(ex))
return
def _validateArgument(argument, value, argumentDefinition):
"""
Validate a single argument against its definition in the spec
"""
validNull = argumentDefinition["null"]
if not validNull and value is None:
raise WMSpecFactoryException("Argument '%s' cannot be None" % argument)
elif value is None:
return value
try:
value = argumentDefinition["type"](value)
except Exception:
msg = "Argument '%s' with value %r, has an incorrect data type: " % (argument, value)
msg += "%s. It must be %s" % (type(value), argumentDefinition["type"])
raise WMSpecFactoryException(msg)
_validateArgFunction(argument, value, argumentDefinition["validate"])
return value
def validatePhEDExSubscription(arguments):
autoApproveList = arguments.get("AutoApproveSubscriptionSites", [])
notAllowedSites = []
for site in autoApproveList:
if site.endswith('_MSS'):
notAllowedSites.append(site)
if len(notAllowedSites) > 0:
msg = "Validation failed: Auto-approval to MSS endpoint not allowed: %s" % notAllowedSites
raise WMSpecFactoryException(msg)
return
def validateSiteLists(arguments):
whiteList = arguments.get("SiteWhitelist", [])
blackList = arguments.get("SiteBlacklist", [])
whiteList = makeList(whiteList)
blackList = makeList(blackList)
res = (set(whiteList) & set(blackList))
if len(res):
msg = "Validation failed: The same site cannot be white and blacklisted: %s" % list(
res)
raise WMSpecFactoryException(msg)
return
def validateSchema(self, schema):
"""
_validateSchema_
Standard StdBase schema validation, plus verification
of the DQMConfigCacheID.
"""
StdBase.validateSchema(self, schema)
if not schema.get("DQMUploadUrl", None):
msg = "DQMUploadUrl parameter has not been provided in the request"
raise WMSpecFactoryException(message=msg)
if not schema.get("DQMConfigCacheID", None):
msg = "DQMConfigCacheID parameter has not been provided in the request"
raise WMSpecFactoryException(message=msg)
couchUrl = schema.get("ConfigCacheUrl", None)
self.validateConfigCacheExists(configID=schema["DQMConfigCacheID"],
couchURL=couchUrl,
couchDBName=schema["CouchDBName"],
getOutputModules=False)
def validateSiteLists(arguments):
whiteList = arguments.get("SiteWhitelist", [])
blackList = arguments.get("SiteBlacklist", [])
whiteList = makeList(whiteList)
blackList = makeList(blackList)
res = (set(whiteList) & set(blackList))
if len(res):
msg = "Validation failed: The same site cannot be white and blacklisted: %s" % list(res)
raise WMSpecFactoryException(msg)
# store the properly formatted values (list instead of string)
arguments["SiteWhitelist"] = whiteList
arguments["SiteBlacklist"] = blackList
return
def _validateArgumentDict(argument, argValue, argumentDefinition):
"""
Validate arguments that carry a dict value type
"""
validNull = argumentDefinition["null"]
if not validNull and None in argValue.values():
raise WMSpecFactoryException("Argument %s can't be None" % argument)
elif all(val is None for val in argValue.values()):
return argValue
for val in argValue.values():
try:
# sigh.. LumiList has a peculiar type validation
if argument == 'LumiList':
val = argumentDefinition["type"](argValue)
break
val = argumentDefinition["type"](val)
except Exception:
raise WMSpecFactoryException("Argument: %s, value: %s type is incorrect in schema." % (argument, val))
_validateArgFunction(argument, argValue, argumentDefinition["validate"])
return argValue
def _validateInputDataset(arguments):
inputdataset = arguments.get("InputDataset", None)
dbsURL = arguments.get("DbsUrl", None)
if inputdataset != None and dbsURL != None:
#import DBS3Reader here, since Runtime code import this module and worker node doesn't have dbs3 client
from WMCore.Services.DBS.DBS3Reader import DBS3Reader
from WMCore.Services.DBS.DBSErrors import DBSReaderError
try:
DBS3Reader(dbsURL).checkDatasetPath(inputdataset)
except DBSReaderError as ex:
# we need to Wrap the exception to WMSpecFactoryException to be caught in reqmgr validation
raise WMSpecFactoryException(str(ex))
return
def strToBool(string):
"""
_strToBool_
Convert the string to the matching boolean value:
i.e. "True" to python True
"""
if string == False or string == True:
return string
# Should we make it more human-friendly (i.e. string in ("Yes", "True", "T")?
elif string == "True":
return True
elif string == "False":
return False
else:
raise WMSpecFactoryException("Can't convert to bool: %s" % string)
def _validateArgumentDict(argument, argValue, argumentDefinition):
"""
Validate only the basic structure of dict arguments, we anyways
don't have the definition of the internal arguments.
"""
# make sure we're not going to cast a dict to string and let that unnoticed
if isinstance(argumentDefinition["type"], type(dict)) and not isinstance(argValue, dict):
msg = "Argument '%s' with value %r, has an incorrect data type: " % (argument, argValue)
msg += "%s. It must be %s" % (type(argValue), argumentDefinition["type"])
raise WMSpecFactoryException(msg)
# still an exception, make sure it has the correct format
if argument == "LumiList":
argValue = argumentDefinition["type"](argValue)
_validateArgFunction(argument, argValue, argumentDefinition["validate"])
return argValue
def makeList(stringList):
"""
_makeList_
Make a python list out of a comma separated list of strings,
throws a WMSpecFactoryException if the input is not
well formed. If the stringList is already of type list
then it is return untouched
"""
if isinstance(stringList, list):
return stringList
if isinstance(stringList, basestring):
toks = stringList.lstrip(' [').rstrip(' ]').split(',')
if toks == ['']:
return []
return [str(tok.strip(' \'"')) for tok in toks]
raise WMSpecFactoryException("Can't convert to list %s" % stringList)
def _validateArgumentOptions(arguments, argumentDefinition, optionKey=None):
"""
Check whether create or assign mandatory parameters were properly
set in the request schema.
"""
for arg, argDef in argumentDefinition.iteritems():
optional = argDef.get(optionKey, True)
if not optional and arg not in arguments:
msg = "Validation failed: %s parameter is mandatory. Definition: %s" % (arg, argDef)
raise WMSpecFactoryException(msg)
# specific case when user GUI returns empty string for optional arguments
elif arg not in arguments:
continue
elif isinstance(arguments[arg], dict):
arguments[arg] = _validateArgumentDict(arg, arguments[arg], argDef)
else:
arguments[arg] = _validateArgument(arg, arguments[arg], argDef)
return
