def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_num("pandajobid", param, safe, optional=False)
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False)
validate_str("outlocation", param, safe, RX_CMSSITE, optional=False)
validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False)
validate_str("acquisitionera", param, safe, RX_WORKFLOW, optional=False)#TODO Do we really need this?
validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_num("events", param, safe, optional=False)
elif method in ['POST']:
raise NotImplementedError
elif method in ['GET']:
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
elif method in ['DELETE']:
raise NotImplementedError
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid() # TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here.
# Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource)
if method in ["POST"]:
validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
validate_str("status", param, safe, RX_STATUS, optional=True)
validate_str("command", param, safe, RX_STATUS, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
validate_numlist("resubmittedjobs", param, safe)
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
validate_num("limit", param, safe, optional=True)
validate_num("clusterid", param, safe, optional=True) # clusterid of the dag
# possible combinations to check
# 1) taskname + status
# 2) taskname + status + failure
# 3) taskname + status + resubmitted + jobsetid
# 4) taskname + status == (1)
# 5) status + limit + getstatus + workername
# 6) taskname + runs + lumis
elif method in ["GET"]:
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("limit", param, safe, optional=True)
validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_str("jobid", param, safe, RX_JOBID, optional=True)
#TODO: for backward compatibility. Get rid of the pandajobid once all jobs using it are done (one month after this gets released)
#(The following five lines can be deleted)
validate_num("pandajobid", param, safe, optional=True)
if bool(safe.kwargs["jobid"]) == bool(safe.kwargs["pandajobid"]):
raise InvalidParameter("Only one among jobid and pandajobid should be set")
#Oracle/cx_oracle/python stack does not like None for numbers, even if they are nullable
if safe.kwargs["pandajobid"] == None:
safe.kwargs["pandajobid"] = 0
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False)
validate_str("outlocation", param, safe, RX_CMSSITE, optional=False)
validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False)
validate_str("acquisitionera", param, safe, RX_TASKNAME, optional=False)#TODO Do we really need this?
validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
validate_num("events", param, safe, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
validate_num("directstageout", param, safe, optional=True)
safe.kwargs["directstageout"] = 'T' if safe.kwargs["directstageout"] else 'F' #'F' if not provided
elif method in ['POST']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=False)
elif method in ['GET']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
validate_num("howmany", param, safe, optional=True)
elif method in ['DELETE']:
authz_operator()
validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
validate_str("hours", param, safe, RX_HOURS, optional=True)
if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
" will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_num("pandajobid", param, safe, optional=False)
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False)
validate_str("outlocation", param, safe, RX_CMSSITE, optional=False)
validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False)
validate_str("acquisitionera", param, safe, RX_WORKFLOW, optional=False)#TODO Do we really need this?
validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
validate_num("events", param, safe, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
validate_num("directstageout", param, safe, optional=True)
safe.kwargs["directstageout"] = 'T' if safe.kwargs["directstageout"] else 'F' #'F' if not provided
elif method in ['POST']:
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=False)
elif method in ['GET']:
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
elif method in ['DELETE']:
authz_operator()
validate_str("taskname", param, safe, RX_WORKFLOW, optional=True)
validate_str("hours", param, safe, RX_HOURS, optional=True)
if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
" will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid() #TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here.
# Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource)
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_num("subjobdef", param, safe, optional=True)
validate_str("substatus", param, safe, RX_STATUS, optional=False)
validate_strlist("subblocks", param, safe, RX_BLOCK)
validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=False)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=True)
validate_str("status", param, safe, RX_STATUS, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("jobset", param, safe, optional=True)
validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
validate_numlist("resubmittedjobs", param, safe)
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
validate_num("limit", param, safe, optional=True)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
# possible combinations to check
# 1) taskname + status
# 2) taskname + status + failure
# 3) taskname + status + resubmitted + jobsetid
# 4) taskname + status == (1)
# 5) status + limit + getstatus + workername
# 6) taskname + runs + lumis
elif method in ['GET']:
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("limit", param, safe, optional=True)
validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
validate_num("subjobdef", param, safe, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=True)
# possible combinations to check
# 1) workername + getstatus + limit
# 2) subresource + subjobdef + subuser
elif method in ['DELETE']:
pass
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_num("subjobdef", param, safe, optional=True)
validate_str("substatus", param, safe, RX_STATUS, optional=False)
validate_strlist("subblocks", param, safe, RX_BLOCK)
validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=False)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=True)
validate_str("status", param, safe, RX_STATUS, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("jobset", param, safe, optional=True)
validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
validate_numlist("resubmittedjobs", param, safe)
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
validate_num("limit", param, safe, optional=True)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
validate_strlist("outputdataset", param, safe, RX_OUT_DATASET)
# possible combinations to check
# 1) taskname + status
# 2) taskname + status + failure
# 3) taskname + status + resubmitted + jobsetid
# 4) taskname + status == (1)
# 5) status + limit + getstatus + workername
# 6) taskname + runs + lumis
elif method in ['GET']:
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("limit", param, safe, optional=True)
validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
validate_num("subjobdef", param, safe, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=True)
# possible combinations to check
# 1) workername + getstatus + limit
# 2) subresource + subjobdef + subuser
elif method in ['DELETE']:
pass
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_num("subjobdef", param, safe, optional=True)
validate_str("substatus", param, safe, RX_STATUS, optional=False)
validate_strlist("subblocks", param, safe, RX_BLOCK)
validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=False)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=True)
validate_str("status", param, safe, RX_STATUS, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("jobset", param, safe, optional=True)
validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
validate_numlist("resubmittedjobs", param, safe)
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
validate_num("limit", param, safe, optional=True)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
# possible combinations to check
# 1) taskname + status
# 2) taskname + status + failure
# 3) taskname + status + resubmitted + jobsetid
# 4) taskname + status == (1)
# 5) status + limit + getstatus + workername
# 6) taskname + runs + lumis
elif method in ['GET']:
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("limit", param, safe, optional=True)
validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
validate_num("subjobdef", param, safe, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=True)
# possible combinations to check
# 1) workername + getstatus + limit
# 2) subresource + subjobdef + subuser
elif method in ['DELETE']:
pass
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
jobtype = safe.kwargs.get('jobtype', None)
if jobtype == 'Analysis':
validate_str("inputdata", param, safe, RX_DATASET, optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
#validate_str("userisburl", param, safe, re.compile(r"^[A-Za-z]*$"), optional=False)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_str("publishname", param, safe, RX_PUBLISH, optional=True)
validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=True)
validate_num("publication", param, safe, optional=False)
#if publication is set as true both publishDataName and publishDbsUrl are needed
if safe.kwargs["publication"] and not (bool(safe.kwargs["publishname"]) and bool(safe.kwargs["publishdbsurl"])):
raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication")
#if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
if bool(safe.kwargs["publishname"]) != bool(safe.kwargs["publishdbsurl"]):
raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication")
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkSite(safe.kwargs['asyncdest'])
validate_num("blacklistT1", param, safe, optional=False)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=True)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
elif method in ['POST']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
elif method in ['GET']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
#parameters of subresources calls has to be put here
#used by get latest
validate_num('age', param, safe, optional=True)
#used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
#used by errors
validate_num('shortformat', param, safe, optional=True)
#validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_num("force", param, safe, optional=True)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
#authz_login_valid()
if method in ['PUT']:
validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
# TODO this should be changed to be non-optional
validate_str("generator", param, safe, RX_GENERATOR, optional=True)
validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_num("nonprodsw", param, safe, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
jobtype = safe.kwargs.get('jobtype', None)
validate_num("useparent", param, safe, optional=True)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_num("totalunits", param, safe, optional=True)
validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
validate_str("lfn", param, safe, RX_LFN, optional=True)
self._checkOutLFN(safe.kwargs)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
validate_strlist("userfiles", param, safe, RX_USERFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_num("saveoutput", param, safe, optional=True)
validate_num("faillimit", param, safe, optional=True)
validate_num("ignorelocality", param, safe, optional=True)
if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_num("publication", param, safe, optional=False)
validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication'])))
## Validations needed in case publication is on.
if safe.kwargs['publication']:
## The (user specified part of the) publication dataset name must be
## specified and must pass DBS validation. Since this is the correct
## validation function, it must be done before the
## validate_str("workflow", ...) and validate_str("publishname", ...)
## we have below. Not sure why RX_PUBLISH was introduced in CRAB, but
## it is not the correct regular expression for publication dataset
## name validation.
self._checkPublishDataName(param.kwargs, safe.kwargs['lfn'])
## 'publishname' was already validated above in _checkPublishDataName().
## But I am not sure if we can skip the validate_str('publishname', ...)
## or we need it so that all parameters are moved from param to safe.
## Therefore, I didn't remove the validate_str('publishname', ...),
## but only changed RX_PUBLISH -> RX_ANYTHING.
validate_str('publishname', param, safe, RX_ANYTHING, optional=False)
## If this is a MC generation job type, the primary dataset name will
## be used to define the primary dataset field in the publication
## dataset name. Therefore, the primary dataset name must pass DBS
## validation. Since in a MC generation job type the primary dataset
## name is saved in 'inputdata', _checkPrimaryDataset() actually
## validates 'inputdata'. Therefore, this function must come before
## the validate_str('inputdata', ...) we have below. OTOH, matching
## 'inputdata' agains RX_DATASET is not the correct validation for
## the primary dataset name.
if jobtype == 'PrivateMC':
self._checkPrimaryDataset(param.kwargs)
else:
## Not sure if we need to match publishname against RX_PUBLISH...
validate_str("publishname", param, safe, RX_PUBLISH, optional=False)
validate_num("publishgroupname", param, safe, optional=True)
## This line must come after _checkPublishDataName()
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
if jobtype == 'Analysis':
validate_str("inputdata", param, safe, RX_DATASET, optional=False)
## Added this if, because in this case 'inputdata' was already validated
## above in _checkPrimaryDataset(), and I am not sure if we can skip the
## validate_str('inputdata', ...) or we need it so that all parameters
## are moved from param to safe.
elif jobtype == 'PrivateMC' and safe.kwargs['publication']:
validate_str("inputdata", param, safe, RX_ANYTHING, optional=False)
else:
validate_str("inputdata", param, safe, RX_DATASET, optional=True)
validate_num("nonvaliddata", param, safe, optional=True)
#if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkASODestination(safe.kwargs['asyncdest'])
# We no longer use this attribute, but keep it around for older client compatibility
validate_num("blacklistT1", param, safe, optional=True)
validate_num("oneEventMode", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
#validate_str("scheduler", param, safe, RX_SCHEDULER)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
validate_str("asourl", param, safe, RX_ASOURL, optional=True)
validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
validate_str("collector", param, safe, RX_COLLECTOR, optional=True)
validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
validate_num("dryrun", param, safe, optional=True)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True)
## In a resubmission, the site black- and whitelists need to be interpreted
## differently than in an initial task submission. If there is no site black-
## or whitelist, set it to None and DataWorkflow will use the corresponding
## list defined in the initial task submission. If the site black- or whitelist
## is equal to the string 'empty', set it to an empty list and don't call
## validate_strlist as it would fail.
if 'siteblacklist' not in param.kwargs:
safe.kwargs['siteblacklist'] = None
elif param.kwargs['siteblacklist'] == 'empty':
safe.kwargs['siteblacklist'] = []
del param.kwargs['siteblacklist']
else:
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
if 'sitewhitelist' not in param.kwargs:
safe.kwargs['sitewhitelist'] = None
elif param.kwargs['sitewhitelist'] == 'empty':
safe.kwargs['sitewhitelist'] = []
del param.kwargs['sitewhitelist']
else:
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_numlist('jobids', param, safe)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_num("force", param, safe, optional=True)
elif method in ['GET']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric
## Used to determine how much information to return to the client for status.
## also used by report to determine if it has to check job states
validate_num("verbose", param, safe, optional=True)
## used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
## used by errors and report (short format in report means we do not query DBS)
validate_num('shortformat', param, safe, optional=True)
## validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_num("force", param, safe, optional=True)
validate_numlist('jobids', param, safe)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid(
) #TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here.
# Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource)
if method in ['PUT']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_num("subjobdef", param, safe, optional=True)
validate_str("substatus", param, safe, RX_STATUS, optional=False)
validate_strlist("subblocks", param, safe, RX_BLOCK)
validate_str("subfailure",
param,
safe,
RX_TEXT_FAIL,
optional=True)
validate_str("subuser", param, safe, RX_DN, optional=False)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
validate_str("status", param, safe, RX_STATUS, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("jobset", param, safe, optional=True)
validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
validate_numlist("resubmittedjobs", param, safe)
validate_str("workername",
param,
safe,
RX_WORKER_NAME,
optional=True)
validate_str("subresource",
param,
safe,
RX_SUBPOSTWORKER,
optional=True)
validate_num("limit", param, safe, optional=True)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
# possible combinations to check
# 1) taskname + status
# 2) taskname + status + failure
# 3) taskname + status + resubmitted + jobsetid
# 4) taskname + status == (1)
# 5) status + limit + getstatus + workername
# 6) taskname + runs + lumis
elif method in ['GET']:
validate_str("workername",
param,
safe,
RX_WORKER_NAME,
optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("limit", param, safe, optional=True)
validate_str("subresource",
param,
safe,
RX_SUBGETWORKER,
optional=True)
validate_num("subjobdef", param, safe, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=True)
# possible combinations to check
# 1) workername + getstatus + limit
# 2) subresource + subjobdef + subuser
elif method in ['DELETE']:
pass
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
#authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
validate_str("generator", param, safe, RX_GENERATOR, optional=True)
validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_num("nonprodsw", param, safe, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
jobtype = safe.kwargs.get('jobtype', None)
if jobtype == 'Analysis':
validate_str("inputdata", param, safe, RX_DATASET, optional=False)
else:
validate_str("inputdata", param, safe, RX_DATASET, optional=True)
validate_num("useparent", param, safe, optional=True)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_num("totalunits", param, safe, optional=True)
validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
validate_str("lfnprefix", param, safe, RX_LFNPATH, optional=True) #keeping this for compatibility with old clients. Will be converted into a full lfn
validate_str("lfn", param, safe, RX_LFN, optional=True)
self._checkOutLFN(safe.kwargs)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
validate_strlist("userfiles", param, safe, RX_USERFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_num("saveoutput", param, safe, optional=True)
validate_num("faillimit", param, safe, optional=True)
validate_num("ignorelocality", param, safe, optional=True)
if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_str("publishname", param, safe, RX_PUBLISH, optional=False)
validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=True)
validate_num("publication", param, safe, optional=False)
#if publication is set as true both publishDataName and publishDbsUrl are needed
if safe.kwargs["publication"] and not (bool(safe.kwargs["publishname"]) and bool(safe.kwargs["publishdbsurl"])):
raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication")
#if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkASODestination(safe.kwargs['asyncdest'])
# We no longer use this attribute, but keep it around for older client compatibility
validate_num("blacklistT1", param, safe, optional=True)
validate_num("oneEventMode", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
#validate_str("scheduler", param, safe, RX_SCHEDULER)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
validate_str("asourl", param, safe, RX_ASOURL, optional=True)
validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_numlist('jobids', param, safe)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
elif method in ['GET']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric
## Used to determine how much information to return to the client for status.
## also used by report to determine if it has to check job states
validate_num("verbose", param, safe, optional=True)
## used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
## used by errors and report (short format in report means we do not query DBS)
validate_num('shortformat', param, safe, optional=True)
## validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_num("force", param, safe, optional=True)
validate_numlist('jobids', param, safe)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
jobtype = safe.kwargs.get('jobtype', None)
if jobtype == 'Analysis':
validate_str("inputdata",
param,
safe,
RX_DATASET,
optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(
safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(
safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_str("cachefilename",
param,
safe,
RX_CACHENAME,
optional=False)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
#validate_str("userisburl", param, safe, re.compile(r"^[A-Za-z]*$"), optional=False)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_str("publishname", param, safe, RX_PUBLISH, optional=True)
validate_str("publishdbsurl",
param,
safe,
RX_DBSURL,
optional=True)
validate_num("publication", param, safe, optional=False)
#if publication is set as true both publishDataName and publishDbsUrl are needed
if safe.kwargs["publication"] and not (
bool(safe.kwargs["publishname"])
and bool(safe.kwargs["publishdbsurl"])):
raise InvalidParameter(
"You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication"
)
#if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
if bool(safe.kwargs["publishname"]) != bool(
safe.kwargs["publishdbsurl"]):
raise InvalidParameter(
"You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication"
)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkSite(safe.kwargs['asyncdest'])
validate_num("blacklistT1", param, safe, optional=False)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=True)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter(
"The number of runs and the number of lumis lists are different"
)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(
safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(
safe.kwargs['sitewhitelist'])
elif method in ['GET']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
validate_str('subresource',
param,
safe,
RX_SUBRESTAT,
optional=True)
#parameters of subresources calls has to be put here
#used by get latest
validate_num('age', param, safe, optional=True)
#used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
#used by errors
validate_num('shortformat', param, safe, optional=True)
#validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in [
'data', 'logs'
] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter(
"You need to specify the number of jobs to retrieve or their ids."
)
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_num("force", param, safe, optional=True)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
#authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
validate_str("generator", param, safe, RX_GENERATOR, optional=True)
validate_str("eventsperlumi",
param,
safe,
RX_LUMIEVENTS,
optional=True)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_num("nonprodsw", param, safe, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
if not safe.kwargs[
"nonprodsw"]: #if the user wants to allow non-production releases
self._checkReleases(safe.kwargs['jobarch'],
safe.kwargs['jobsw'])
jobtype = safe.kwargs.get('jobtype', None)
if jobtype == 'Analysis':
validate_str("inputdata",
param,
safe,
RX_DATASET,
optional=False)
else:
validate_str("inputdata",
param,
safe,
RX_DATASET,
optional=True)
validate_num("useparent", param, safe, optional=True)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(
safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(
safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_num("totalunits", param, safe, optional=True)
validate_str("cachefilename",
param,
safe,
RX_CACHENAME,
optional=False)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
validate_str(
"lfnprefix", param, safe, RX_LFNPATH, optional=True
) #keeping this for compatibility with old clients. Will be converted into a full lfn
validate_str("lfn", param, safe, RX_LFN, optional=True)
self._checkOutLFN(safe.kwargs)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
validate_strlist("userfiles", param, safe, RX_USERFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_num("saveoutput", param, safe, optional=True)
validate_num("faillimit", param, safe, optional=True)
validate_num("ignorelocality", param, safe, optional=True)
if safe.kwargs[
'ignorelocality'] and self.centralcfg.centralconfig.get(
'ign-locality-blacklist', []):
safe.kwargs['siteblacklist'] += self._expandSites(
self.centralcfg.centralconfig['ign-locality-blacklist'])
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_str("publishname",
param,
safe,
RX_PUBLISH,
optional=False)
validate_str("publishdbsurl",
param,
safe,
RX_DBSURL,
optional=True)
validate_num("publication", param, safe, optional=False)
#if publication is set as true both publishDataName and publishDbsUrl are needed
if safe.kwargs["publication"] and not (
bool(safe.kwargs["publishname"])
and bool(safe.kwargs["publishdbsurl"])):
raise InvalidParameter(
"You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication"
)
#if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkASODestination(safe.kwargs['asyncdest'])
# We no longer use this attribute, but keep it around for older client compatibility
validate_num("blacklistT1", param, safe, optional=True)
validate_num("oneEventMode", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
#validate_str("scheduler", param, safe, RX_SCHEDULER)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter(
"The number of runs and the number of lumis lists are different"
)
validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
validate_str("asourl", param, safe, RX_ASOURL, optional=True)
validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
validate_str("scheddname",
param,
safe,
RX_SCHEDD_NAME,
optional=True)
validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(
safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(
safe.kwargs['sitewhitelist'])
validate_numlist('jobids', param, safe)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
elif method in ['GET']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
validate_str('subresource',
param,
safe,
RX_SUBRESTAT,
optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
validate_str('timestamp', param, safe, RX_DATE,
optional=True) ## inserted by eric
## Used to determine how much information to return to the client for status.
## also used by report to determine if it has to check job states
validate_num("verbose", param, safe, optional=True)
## used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
## used by errors and report (short format in report means we do not query DBS)
validate_num('shortformat', param, safe, optional=True)
## validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in [
'data', 'logs'
] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter(
"You need to specify the number of jobs to retrieve or their ids."
)
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_num("force", param, safe, optional=True)
validate_numlist('jobids', param, safe)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(
safe.kwargs["outfilelumis"]):
raise InvalidParameter(
"The number of runs and the number of lumis lists are different"
)
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_num("pandajobid", param, safe, optional=False)
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname",
param,
safe,
RX_PUBLISH,
optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumcksum",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumadler32",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("outlocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("outtmplocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("acquisitionera",
param,
safe,
RX_WORKFLOW,
optional=False) #TODO Do we really need this?
validate_str(
"outdatasetname", param, safe, RX_OUTDSLFN,
optional=False) #TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_num("events", param, safe, optional=False)
elif method in ['POST']:
raise NotImplementedError
elif method in ['GET']:
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
elif method in ['DELETE']:
raise NotImplementedError
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(
safe.kwargs["outfilelumis"]):
raise InvalidParameter(
"The number of runs and the number of lumis lists are different"
)
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_num("pandajobid", param, safe, optional=False)
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname",
param,
safe,
RX_PUBLISH,
optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumcksum",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumadler32",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("outlocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("outtmplocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("acquisitionera",
param,
safe,
RX_TASKNAME,
optional=False) #TODO Do we really need this?
validate_str(
"outdatasetname", param, safe, RX_OUTDSLFN,
optional=False) #TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
validate_num("events", param, safe, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
validate_num("directstageout", param, safe, optional=True)
safe.kwargs["directstageout"] = 'T' if safe.kwargs[
"directstageout"] else 'F' #'F' if not provided
elif method in ['POST']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_str("filestate",
param,
safe,
RX_FILESTATE,
optional=False)
elif method in ['GET']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
elif method in ['DELETE']:
authz_operator()
validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
validate_str("hours", param, safe, RX_HOURS, optional=True)
if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
" will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(
safe.kwargs["outfilelumis"]):
raise InvalidParameter(
"The number of runs and the number of lumis lists are different"
)
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_str("jobid", param, safe, RX_JOBID, optional=True)
#TODO: for backward compatibility. Get rid of the pandajobid once all jobs using it are done (one month after this gets released)
#(The following five lines can be deleted)
validate_num("pandajobid", param, safe, optional=True)
if bool(safe.kwargs["jobid"]) == bool(safe.kwargs["pandajobid"]):
raise InvalidParameter(
"Only one among jobid and pandajobid should be set")
#Oracle/cx_oracle/python stack does not like None for numbers, even if they are nullable
if safe.kwargs["pandajobid"] == None:
safe.kwargs["pandajobid"] = 0
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname",
param,
safe,
RX_PUBLISH,
optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumcksum",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumadler32",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("outlocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("outtmplocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("acquisitionera",
param,
safe,
RX_TASKNAME,
optional=False) #TODO Do we really need this?
validate_str(
"outdatasetname", param, safe, RX_OUTDSLFN,
optional=False) #TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
validate_num("events", param, safe, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
validate_num("directstageout", param, safe, optional=True)
safe.kwargs["directstageout"] = 'T' if safe.kwargs[
"directstageout"] else 'F' #'F' if not provided
elif method in ['POST']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_str("filestate",
param,
safe,
RX_FILESTATE,
optional=False)
elif method in ['GET']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
validate_num("howmany", param, safe, optional=True)
validate_strlist("lfn", param, safe, RX_LFN)
elif method in ['DELETE']:
authz_operator()
validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
validate_str("hours", param, safe, RX_HOURS, optional=True)
if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
" will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
if method in ['PUT']:
username = cherrypy.request.user['login'] # username registered in SiteDB
requestname = param.kwargs['workflow']
param.kwargs['workflow'] = generateTaskName(username, requestname)
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
# TODO this should be changed to be non-optional
validate_str("generator", param, safe, RX_GENERATOR, optional=True)
validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_num("nonprodsw", param, safe, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
validate_num("useparent", param, safe, optional=True)
validate_str("secondarydata", param, safe, RX_DATASET, optional=True)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_num("totalunits", param, safe, optional=True)
validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
validate_str("debugfilename", param, safe, RX_CACHENAME, optional=True)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
validate_str("lfn", param, safe, RX_LFN, optional=True)
self._checkOutLFN(safe.kwargs, username)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE, custom_err="Incorrect 'JobType.outputFiles' parameter. " \
"Allowed regexp: '%s'." % RX_ADDFILE.pattern)
validate_strlist("userfiles", param, safe, RX_USERFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_num("saveoutput", param, safe, optional=True)
validate_num("faillimit", param, safe, optional=True)
validate_num("ignorelocality", param, safe, optional=True)
if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_num("publication", param, safe, optional=False)
validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication'])))
## We might want to remove publishname once the backward compatibility
## wont be needed anymore. Then we can just keep publishname2
## NB: AFAIK the only client not using the CRABLibrary but direct cutl is HC,
## therefore we will need to make sure we do not break it!
## The following two lines will be removed in the future once we will
## not need backward compatibility anymore
self._checkPublishDataName(param.kwargs, safe.kwargs['lfn'], requestname, username)
validate_str('publishname', param, safe, RX_ANYTHING, optional=True)
##And this if as well, just do self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
if not safe.kwargs["publishname"]: #new clients won't define this anymore
## The (user specified part of the) publication dataset name must be
## specified and must pass DBS validation. Since this is the correct
## validation function, it must be done before the
## validate_str("publishname", ...) we have below.
self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
else:
param.kwargs["publishname2"] = safe.kwargs["publishname"]
## 'publishname' was already validated above in _checkPublishDataName().
## Calling validate_str with a fake regexp to move the param to the
## list of validated inputs
validate_str("publishname2", param, safe, RX_ANYTHING, optional=True)
validate_num("publishgroupname", param, safe, optional=True)
if safe.kwargs['jobtype'] == 'PrivateMC':
if param.kwargs['inputdata']:
msg = "Invalid 'inputdata' parameter."
msg += " Job type PrivateMC does not take any input dataset."
msg += " If you really intend to run over an input dataset, then you must use job type Analysis."
raise InvalidParameter(msg)
if safe.kwargs['userfiles']:
msg = "Invalid 'userfiles' parameter."
msg += " Job type PrivateMC does not take any input files."
msg += " If you really intend to run over input files, then you must use job type Analysis."
raise InvalidParameter(msg)
## Client versions < 3.3.1511 may put in the input dataset something that is not
## really an input dataset (for PrivateMC or user input files). So the only case
## in which we are sure that we have to validate the input dataset is when the
## workflow type is Analysis, the workflow does not run on user input files and
## an input dataset is defined (scriptExe may not define an input).
## Once we don't care anymore about backward compatibility with client < 3.3.1511,
## we can uncomment the 1st line below and delete the next 4 lines.
#validate_str("inputdata", param, safe, RX_DATASET, optional=True)
if safe.kwargs['jobtype'] == 'Analysis' and not safe.kwargs['userfiles'] and 'inputdata' in param.kwargs:
validate_str("inputdata", param, safe, RX_DATASET, optional=True)
else:
validate_str("inputdata", param, safe, RX_ANYTHING, optional=True)
## The client is not forced to define the primary dataset. So make sure to have
## defaults or take it from the input dataset. The primary dataset is needed for
## the LFN of the output/log files and for publication. We want to have it well
## defined even if publication and/or transfer to storage are off.
if safe.kwargs['inputdata']:
param.kwargs['primarydataset'] = safe.kwargs['inputdata'].split('/')[1]
if not param.kwargs.get('primarydataset', None):
if safe.kwargs['jobtype'] == 'PrivateMC':
param.kwargs['primarydataset'] = "CRAB_PrivateMC"
elif safe.kwargs['jobtype'] == 'Analysis' and safe.kwargs['userfiles']:
param.kwargs['primarydataset'] = "CRAB_UserFiles"
else:
param.kwargs['primarydataset'] = "CRAB_NoInput"
## We validate the primary dataset agains DBS rules even if publication is off,
## because in the future we may want to give the possibility to users to publish
## a posteriori.
self._checkPrimaryDataset(param.kwargs, optional=False)
validate_str("primarydataset", param, safe, RX_LFNPRIMDS, optional=False)
validate_num("nonvaliddata", param, safe, optional=True)
#if one and only one between outputDatasetTag and publishDbsUrl is set raise an error (we need both or none of them)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkASODestination(safe.kwargs['asyncdest'])
# We no longer use this attribute, but keep it around for older client compatibility
validate_num("blacklistT1", param, safe, optional=True)
validate_num("oneEventMode", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect tfileoutfiles parameter (TFileService). " \
"Allowed regexp: '%s'." % RX_OUTFILES.pattern)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect edmoutfiles parameter (PoolOutputModule). " \
"Allowed regexp: '%s'." % RX_OUTFILES.pattern)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
#validate_str("scheduler", param, safe, RX_SCHEDULER)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
validate_str("asourl", param, safe, RX_ASOURL, optional=True)
validate_str("asodb", param, safe, RX_ASODB, optional=True)
safe.kwargs["asourl"], safe.kwargs["asodb"] = self._getAsoConfig(safe.kwargs["asourl"], safe.kwargs["asodb"])
validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
validate_str("collector", param, safe, RX_COLLECTOR, optional=True)
validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
validate_num("dryrun", param, safe, optional=True)
validate_num("ignoreglobalblacklist", param, safe, optional=True)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True)
validate_numlist('jobids', param, safe)
## In a resubmission, the site black- and whitelists need to be interpreted
## differently than in an initial task submission. If there is no site black-
## or whitelist, set it to None and DataWorkflow will use the corresponding
## list defined in the initial task submission. If the site black- or whitelist
## is equal to the string 'empty', set it to an empty list and don't call
## validate_strlist as it would fail.
if 'siteblacklist' not in param.kwargs:
safe.kwargs['siteblacklist'] = None
elif param.kwargs['siteblacklist'] == 'empty':
safe.kwargs['siteblacklist'] = []
del param.kwargs['siteblacklist']
else:
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
if 'sitewhitelist' not in param.kwargs:
safe.kwargs['sitewhitelist'] = None
elif param.kwargs['sitewhitelist'] == 'empty':
safe.kwargs['sitewhitelist'] = []
del param.kwargs['sitewhitelist']
else:
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("force", param, safe, optional=True)
validate_num("publication", param, safe, optional=True)
elif method in ['GET']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric
## Used to determine how much information to return to the client for status.
## also used by report to determine if it has to check job states
validate_num("verbose", param, safe, optional=True)
## used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
## used by errors and report (short format in report means we do not query DBS)
validate_num('shortformat', param, safe, optional=True)
## validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_num("force", param, safe, optional=True)
validate_numlist('jobids', param, safe)
validate_str("killwarning", param, safe, RX_TEXT_FAIL, optional=True)
#decode killwarning message if present
if safe.kwargs['killwarning']:
try:
safe.kwargs['killwarning'] = b64decode(safe.kwargs['killwarning'])
except TypeError:
raise InvalidParameter("Failure message is not in the accepted format")
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
#authz_login_valid()
if method in ['PUT']:
## Define the taskname and write it in the 'workflow' parameter.
timestamp = time.strftime('%y%m%d_%H%M%S', time.gmtime())
username = cherrypy.request.user['login'] # username registered in SiteDB
requestname = param.kwargs['workflow']
taskname = "%s:%s_%s" % (timestamp, username, requestname)
param.kwargs['workflow'] = taskname
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
# TODO this should be changed to be non-optional
validate_str("generator", param, safe, RX_GENERATOR, optional=True)
validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_num("nonprodsw", param, safe, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
validate_num("useparent", param, safe, optional=True)
validate_str("secondarydata", param, safe, RX_DATASET, optional=True)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_num("totalunits", param, safe, optional=True)
validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
validate_str("lfn", param, safe, RX_LFN, optional=True)
self._checkOutLFN(safe.kwargs, username)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
validate_strlist("userfiles", param, safe, RX_USERFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_num("saveoutput", param, safe, optional=True)
validate_num("faillimit", param, safe, optional=True)
validate_num("ignorelocality", param, safe, optional=True)
if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_num("publication", param, safe, optional=False)
validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication'])))
## We might want to remove publishname once the backward compatibility
## wont be needed anymore. Then we can just keep publishname2
## NB: AFAIK the only client not using the CRABLibrary but direct cutl is HC,
## therefore we will need to make sure we do not break it!
## The following two lines will be removed in the future once we will
## not need backward compatibility anymore
self._checkPublishDataName(param.kwargs, safe.kwargs['lfn'], requestname, username)
validate_str('publishname', param, safe, RX_ANYTHING, optional=True)
##And this if as well, just do self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
if not safe.kwargs["publishname"]: #new clients won't define this anymore
## The (user specified part of the) publication dataset name must be
## specified and must pass DBS validation. Since this is the correct
## validation function, it must be done before the
## validate_str("publishname", ...) we have below.
self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
else:
param.kwargs["publishname2"] = safe.kwargs["publishname"]
## 'publishname' was already validated above in _checkPublishDataName().
## Calling validate_str with a fake regexp to move the param to the
## list of validated inputs
validate_str("publishname2", param, safe, RX_ANYTHING, optional=True)
validate_num("publishgroupname", param, safe, optional=True)
if safe.kwargs['jobtype'] == 'PrivateMC':
if param.kwargs['inputdata']:
msg = "Invalid 'inputdata' parameter."
msg += " Job type PrivateMC does not take any input dataset."
msg += " If you really intend to run over an input dataset, then you must use job type Analysis."
raise InvalidParameter(msg)
if safe.kwargs['userfiles']:
msg = "Invalid 'userfiles' parameter."
msg += " Job type PrivateMC does not take any input files."
msg += " If you really intend to run over input files, then you must use job type Analysis."
raise InvalidParameter(msg)
## Client versions < 3.3.1511 may put in the input dataset something that is not
## really an input dataset (for PrivateMC or user input files). So the only case
## in which we are sure that we have to validate the input dataset is when the
## workflow type is Analysis, the workflow does not run on user input files and
## an input dataset is defined (scriptExe may not define an input).
## Once we don't care anymore about backward compatibility with client < 3.3.1511,
## we can uncomment the 1st line below and delete the next 4 lines.
#validate_str("inputdata", param, safe, RX_DATASET, optional=True)
if safe.kwargs['jobtype'] == 'Analysis' and not safe.kwargs['userfiles'] and 'inputdata' in param.kwargs:
validate_str("inputdata", param, safe, RX_DATASET, optional=True)
else:
validate_str("inputdata", param, safe, RX_ANYTHING, optional=True)
## The client is not forced to define the primary dataset. So make sure to have
## defaults or take it from the input dataset. The primary dataset is needed for
## the LFN of the output/log files and for publication. We want to have it well
## defined even if publication and/or transfer to storage are off.
if safe.kwargs['inputdata']:
param.kwargs['primarydataset'] = safe.kwargs['inputdata'].split('/')[1]
if not param.kwargs.get('primarydataset', None):
if safe.kwargs['jobtype'] == 'PrivateMC':
param.kwargs['primarydataset'] = "CRAB_PrivateMC"
elif safe.kwargs['jobtype'] == 'Analysis' and safe.kwargs['userfiles']:
param.kwargs['primarydataset'] = "CRAB_UserFiles"
else:
param.kwargs['primarydataset'] = "CRAB_NoInput"
## We validate the primary dataset agains DBS rules even if publication is off,
## because in the future we may want to give the possibility to users to publish
## a posteriori.
self._checkPrimaryDataset(param.kwargs, optional=False)
validate_str("primarydataset", param, safe, RX_LFNPRIMDS, optional=False)
validate_num("nonvaliddata", param, safe, optional=True)
#if one and only one between outputDatasetTag and publishDbsUrl is set raise an error (we need both or none of them)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkASODestination(safe.kwargs['asyncdest'])
# We no longer use this attribute, but keep it around for older client compatibility
validate_num("blacklistT1", param, safe, optional=True)
validate_num("oneEventMode", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
#validate_str("scheduler", param, safe, RX_SCHEDULER)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
validate_str("asourl", param, safe, RX_ASOURL, optional=True)
validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
validate_str("collector", param, safe, RX_COLLECTOR, optional=True)
validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
validate_num("dryrun", param, safe, optional=True)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True)
validate_numlist('jobids', param, safe)
## In a resubmission, the site black- and whitelists need to be interpreted
## differently than in an initial task submission. If there is no site black-
## or whitelist, set it to None and DataWorkflow will use the corresponding
## list defined in the initial task submission. If the site black- or whitelist
## is equal to the string 'empty', set it to an empty list and don't call
## validate_strlist as it would fail.
if 'siteblacklist' not in param.kwargs:
safe.kwargs['siteblacklist'] = None
elif param.kwargs['siteblacklist'] == 'empty':
safe.kwargs['siteblacklist'] = []
del param.kwargs['siteblacklist']
else:
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
if 'sitewhitelist' not in param.kwargs:
safe.kwargs['sitewhitelist'] = None
elif param.kwargs['sitewhitelist'] == 'empty':
safe.kwargs['sitewhitelist'] = []
del param.kwargs['sitewhitelist']
else:
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("force", param, safe, optional=True)
validate_num("publication", param, safe, optional=True)
elif method in ['GET']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric
## Used to determine how much information to return to the client for status.
## also used by report to determine if it has to check job states
validate_num("verbose", param, safe, optional=True)
## used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
## used by errors and report (short format in report means we do not query DBS)
validate_num('shortformat', param, safe, optional=True)
## validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_num("force", param, safe, optional=True)
validate_numlist('jobids', param, safe)