def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_num("pandajobid", param, safe, optional=False)
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False)
validate_str("outlocation", param, safe, RX_CMSSITE, optional=False)
validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False)
validate_str("acquisitionera", param, safe, RX_WORKFLOW, optional=False)#TODO Do we really need this?
validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_num("events", param, safe, optional=False)
elif method in ['POST']:
raise NotImplementedError
elif method in ['GET']:
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
elif method in ['DELETE']:
raise NotImplementedError
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['POST']:
validate_str('subresource',
param,
safe,
RX_SUBRES_TASK,
optional=False)
validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
validate_str("warning", param, safe, RX_TEXT_FAIL, optional=True)
validate_str("webdirurl", param, safe, RX_URL, optional=True)
validate_str("scheddname",
param,
safe,
RX_SCHEDD_NAME,
optional=True)
validate_strlist("outputdatasets", param, safe, RX_OUT_DATASET)
elif method in ['GET']:
validate_str('subresource',
param,
safe,
RX_SUBRES_TASK,
optional=False)
validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
validate_str('taskstatus', param, safe, RX_STATUS, optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
validate_str('minutes', param, safe, RX_RUNS, optional=True)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False)
validate_strlist("workflow", param, safe, RX_TASKNAME)
elif method in ['POST']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False)
elif method in ['GET']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=True)
validate_num('age', param, safe, optional=True)
validate_str('subresource',
param,
safe,
RX_SUBRESTAT,
optional=True)
validate_num('limit', param, safe, optional=True)
if not safe.kwargs['campaign'] and not safe.kwargs['age']:
raise InvalidParameter("Invalid input parameters")
if not safe.kwargs['campaign'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
elif method in ['DELETE']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False)
validate_num("force", param, safe, optional=True)
def validate(self, apiobj, method, api, param, safe): #pylint: disable=unused-argument
"""Validating all the input parameter as enforced by the WMCore.REST module"""
# TODO, use authz_user_action() which will allow only user to maintain and modify its own
# documents
authz_login_valid()
if method in ['PUT']:
# Do we want to validate everything?
# Now what is put in CouchDB is not validated
# And all put are prepared by us in JOB wrappers, so it should already be correct.
# P.S. Validation is done in function and it double check if all required keys are available
print (param, safe)
validate_str("id", param, safe, RX_ANYTHING, optional=False)
validate_str("username", param, safe, RX_ANYTHING, optional=False)
validate_str("taskname", param, safe, RX_ANYTHING, optional=False)
validate_str("destination", param, safe, RX_ANYTHING, optional=False)
validate_str("destination_lfn", param, safe, RX_ANYTHING, optional=False)
validate_str("source", param, safe, RX_ANYTHING, optional=False)
validate_str("source_lfn", param, safe, RX_ANYTHING, optional=False)
validate_num("filesize", param, safe, optional=False)
validate_num("publish", param, safe, optional=False)
validate_str("job_id", param, safe, RX_JOBID, optional=False)
validate_num("job_retry_count", param, safe, optional=False)
validate_str("type", param, safe, RX_ANYTHING, optional=False)
validate_str("asoworker", param, safe, RX_ANYTHING, optional=True)
validate_num("transfer_retry_count", param, safe, optional=True)
validate_num("transfer_max_retry_count", param, safe, optional=True)
validate_num("publication_retry_count", param, safe, optional=True)
validate_num("publication_max_retry_count", param, safe, optional=True)
validate_num("start_time", param, safe, optional=False)
validate_str("rest_host", param, safe, RX_ANYTHING, optional=False)
validate_str("rest_uri", param, safe, RX_ANYTHING, optional=False)
validate_str("transfer_state", param, safe, RX_ANYTHING, optional=False)
validate_str("publication_state", param, safe, RX_ANYTHING, optional=False)
validate_str("fts_id", param, safe, RX_ANYTHING, optional=True)
validate_str("fts_instance", param, safe, RX_ANYTHING, optional=True)
if method in ['POST']:
# POST is for update, so we should allow anyone anything?
# Of Course no, but there are multiple combinations, so we are not validating here
# and all validation is in post function
validate_str("subresource", param, safe, RX_SUBPOSTUSERTRANSFER, optional=False)
validate_str("id", param, safe, RX_ANYTHING, optional=True)
validate_str("username", param, safe, RX_USERNAME, optional=True)
validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
validate_num("start_time", param, safe, optional=True)
validate_str("source", param, safe, RX_ANYTHING, optional=True)
validate_str("source_lfn", param, safe, RX_ANYTHING, optional=True)
validate_num("filesize", param, safe, optional=True)
validate_str("transfer_state", param, safe, RX_ANYTHING, optional=True)
validate_num("transfer_retry_count", param, safe, optional=True)
validate_str("publication_state", param, safe, RX_ANYTHING, optional=True)
validate_str("job_id", param, safe, RX_JOBID, optional=True)
validate_num("job_retry_count", param, safe, optional=True)
validate_strlist("listOfIds", param, safe, RX_ANYTHING) # Interesting... TODO. Have optional in strlist
elif method in ['GET']:
validate_str("subresource", param, safe, RX_SUBGETUSERTRANSFER, optional=False)
validate_str("id", param, safe, RX_TASKNAME, optional=True)
validate_str("username", param, safe, RX_USERNAME, optional=True)
validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
elif method in ['DELETE']:
raise UnsupportedMethod('This method is not supported in this API!')
def validate(apiobj, method, api, param, safe): #pylint: disable=unused-argument
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid() #TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here.
# Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource)
if method in ['POST']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
validate_str("status", param, safe, RX_STATUS, optional=True)
validate_str("command", param, safe, RX_STATUS, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
validate_strlist("resubmittedjobs", param, safe, RX_JOBID)
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
validate_num("limit", param, safe, optional=True)
validate_num("clusterid", param, safe, optional=True) #clusterid of the dag
# possible combinations to check
# 1) taskname + status
# 2) taskname + status + failure
# 3) taskname + status + resubmitted + jobsetid
# 4) taskname + status == (1)
# 5) status + limit + getstatus + workername
# 6) taskname + runs + lumis
elif method in ['GET']:
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("limit", param, safe, optional=True)
validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_str("jobid", param, safe, RX_JOBID, optional=True)
#TODO: for backward compatibility. Get rid of the pandajobid once all jobs using it are done (one month after this gets released)
#(The following five lines can be deleted)
validate_num("pandajobid", param, safe, optional=True)
if bool(safe.kwargs["jobid"]) == bool(safe.kwargs["pandajobid"]):
raise InvalidParameter("Only one among jobid and pandajobid should be set")
#Oracle/cx_oracle/python stack does not like None for numbers, even if they are nullable
if safe.kwargs["pandajobid"] == None:
safe.kwargs["pandajobid"] = 0
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False)
validate_str("outlocation", param, safe, RX_CMSSITE, optional=False)
validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False)
validate_str("acquisitionera", param, safe, RX_TASKNAME, optional=False)#TODO Do we really need this?
validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
validate_num("events", param, safe, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
validate_num("directstageout", param, safe, optional=True)
safe.kwargs["directstageout"] = 'T' if safe.kwargs["directstageout"] else 'F' #'F' if not provided
elif method in ['POST']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=False)
elif method in ['GET']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
validate_num("howmany", param, safe, optional=True)
elif method in ['DELETE']:
authz_operator()
validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
validate_str("hours", param, safe, RX_HOURS, optional=True)
if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
" will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['POST']:
validate_str('subresource', param, safe, RX_SUBRES_TASK, optional=False)
validate_str("workflow", param, safe, RX_WORKFLOW, optional=True)
validate_str("warning", param, safe, RX_TEXT_FAIL, optional=True)
validate_str("webdirurl", param, safe, RX_URL, optional=True)
validate_strlist("outputdatasets", param, safe, RX_OUT_DATASET)
elif method in ['GET']:
validate_str('subresource', param, safe, RX_SUBRES_TASK, optional=False)
validate_str("workflow", param, safe, RX_WORKFLOW, optional=True)
validate_str('taskstatus', param, safe, RX_STATUS, optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
def validate(apiobj, method, api, param, safe): #pylint: disable=unused-argument
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid(
) #TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here.
# Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource)
if method in ['POST']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
validate_str("status", param, safe, RX_STATUS, optional=True)
validate_str("command", param, safe, RX_STATUS, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
validate_strlist("resubmittedjobs", param, safe, RX_JOBID)
validate_str("workername",
param,
safe,
RX_WORKER_NAME,
optional=True)
validate_str("subresource",
param,
safe,
RX_SUBPOSTWORKER,
optional=True)
validate_num("limit", param, safe, optional=True)
validate_num("clusterid", param, safe,
optional=True) #clusterid of the dag
# possible combinations to check
# 1) taskname + status
# 2) taskname + status + failure
# 3) taskname + status + resubmitted + jobsetid
# 4) taskname + status == (1)
# 5) status + limit + getstatus + workername
# 6) taskname + runs + lumis
elif method in ['GET']:
validate_str("workername",
param,
safe,
RX_WORKER_NAME,
optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("limit", param, safe, optional=True)
validate_str("subresource",
param,
safe,
RX_SUBGETWORKER,
optional=True)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_num("pandajobid", param, safe, optional=False)
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False)
validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False)
validate_str("outlocation", param, safe, RX_CMSSITE, optional=False)
validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False)
validate_str("acquisitionera", param, safe, RX_WORKFLOW, optional=False)#TODO Do we really need this?
validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
validate_num("events", param, safe, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
validate_num("directstageout", param, safe, optional=True)
safe.kwargs["directstageout"] = 'T' if safe.kwargs["directstageout"] else 'F' #'F' if not provided
elif method in ['POST']:
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=False)
elif method in ['GET']:
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
elif method in ['DELETE']:
authz_operator()
validate_str("taskname", param, safe, RX_WORKFLOW, optional=True)
validate_str("hours", param, safe, RX_HOURS, optional=True)
if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
" will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe):
if method == "GET":
if not len(param.args) or param.args[0] not in ("ip", "name", "stats"):
raise InvalidParameter("Missing or wrong ip/host category")
safe.kwargs["kind"] = kind = param.args.pop(0)
if kind == "ip":
validate_iplist("host", param, safe)
elif kind == "name":
validate_strlist("host", param, safe, RXHOST)
elif kind == "stats":
authz_match(role=["Global Admin"], group=["global"])
safe.kwargs["host"] = []
elif method == "POST":
authz_match(role=["Global Admin"], group=["global"])
if not len(param.args) or param.args[0] not in ("stats", "purge"):
raise InvalidParameter("Invalid operation")
safe.kwargs["operation"] = param.args.pop(0)
def validate(self, apiobj, method, api, param, safe):
if method == "GET":
if not len(param.args) or param.args[0] not in ("ip", "name",
"stats"):
raise InvalidParameter("Missing or wrong ip/host category")
safe.kwargs["kind"] = kind = param.args.pop(0)
if kind == "ip":
validate_iplist("host", param, safe)
elif kind == "name":
validate_strlist("host", param, safe, RXHOST)
elif kind == "stats":
authz_match(role=["Global Admin"], group=["global"])
safe.kwargs["host"] = []
elif method == "POST":
authz_match(role=["Global Admin"], group=["global"])
if not len(param.args) or param.args[0] not in ("stats", "purge"):
raise InvalidParameter("Invalid operation")
safe.kwargs["operation"] = param.args.pop(0)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_num("subjobdef", param, safe, optional=True)
validate_str("substatus", param, safe, RX_STATUS, optional=False)
validate_strlist("subblocks", param, safe, RX_BLOCK)
validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=False)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=True)
validate_str("status", param, safe, RX_STATUS, optional=False)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("jobset", param, safe, optional=True)
validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
validate_numlist("resubmittedjobs", param, safe)
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
validate_num("limit", param, safe, optional=True)
# possible combinations to check
# 1) taskname + status
# 2) taskname + status + failure
# 3) taskname + status + resubmitted + jobsetid
# 4) taskname + status == (1)
# 5) status + limit + getstatus + workername
elif method in ['GET']:
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("limit", param, safe, optional=True)
validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
validate_num("subjobdef", param, safe, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=True)
# possible combinations to check
# 1) workername + getstatus + limit
# 2) subresource + subjobdef + subuser
elif method in ['DELETE']:
pass
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False)
validate_strlist("workflow", param, safe, RX_WORKFLOW)
elif method in ['POST']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False)
elif method in ['GET']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=True)
validate_num('age', param, safe, optional=True)
validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
validate_num('limit', param, safe, optional=True)
if not safe.kwargs['campaign'] and not safe.kwargs['age']:
raise InvalidParameter("Invalid input parameters")
if not safe.kwargs['campaign'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
elif method in ['DELETE']:
validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False)
validate_num("force", param, safe, optional=True)
def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
if method == 'GET':
# Check for `performance` endpoint, documented in
# https://github.com/knly/WMArchiveAggregation
if len(param.args) == 1 and param.args[0] == 'performance':
safe.args.append(param.args[0])
param.args.remove(param.args[0])
# Validate arguments
validate_strlist('metrics[]', param, safe, re.compile(r'^[a-zA-Z.]+'))
validate_strlist('axes[]', param, safe, re.compile(r'^[a-zA-Z_]+'))
validate_strlist('suggestions[]', param, safe, re.compile(r'^[a-zA-Z]+'))
date_pattern = PAT_YYYYMMDD
validate_str('start_date', param, safe, date_pattern, optional=True)
validate_str('end_date', param, safe, date_pattern, optional=True)
validate_rx('workflow', param, safe, optional=True)
validate_rx('task', param, safe, optional=True)
validate_rx('host', param, safe, optional=True)
validate_rx('site', param, safe, optional=True)
validate_rx('jobtype', param, safe, optional=True)
validate_rx('jobstate', param, safe, optional=True)
validate_rx('acquisitionEra', param, safe, optional=True)
validate_rx('exitCode', param, safe, optional=True)
validate_rx('exitStep', param, safe, optional=True)
validate_str('_', param, safe, PAT_INFO, optional=True)
return True
if 'query' in param.kwargs.keys():
validate_str('query', param, safe, PAT_QUERY, optional=True)
for key in ['status', 'jobs', '_']:
if key in param.kwargs.keys():
validate_str(key, param, safe, PAT_INFO, optional=True)
# test if user provided uid
if len(param.args) == 1 and PAT_UID.match(param.args[0]):
safe.args.append(param.args[0])
param.args.remove(param.args[0])
return True
elif method == 'POST':
if not param.args or not param.kwargs:
return False # this class does not need any parameters
return True
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid() #TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here.
# Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource)
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_num("subjobdef", param, safe, optional=True)
validate_str("substatus", param, safe, RX_STATUS, optional=False)
validate_strlist("subblocks", param, safe, RX_BLOCK)
validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=False)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=True)
validate_str("status", param, safe, RX_STATUS, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("jobset", param, safe, optional=True)
validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
validate_numlist("resubmittedjobs", param, safe)
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
validate_num("limit", param, safe, optional=True)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
# possible combinations to check
# 1) taskname + status
# 2) taskname + status + failure
# 3) taskname + status + resubmitted + jobsetid
# 4) taskname + status == (1)
# 5) status + limit + getstatus + workername
# 6) taskname + runs + lumis
elif method in ['GET']:
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("limit", param, safe, optional=True)
validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
validate_num("subjobdef", param, safe, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=True)
# possible combinations to check
# 1) workername + getstatus + limit
# 2) subresource + subjobdef + subuser
elif method in ['DELETE']:
pass
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_num("subjobdef", param, safe, optional=True)
validate_str("substatus", param, safe, RX_STATUS, optional=False)
validate_strlist("subblocks", param, safe, RX_BLOCK)
validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=False)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=True)
validate_str("status", param, safe, RX_STATUS, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("jobset", param, safe, optional=True)
validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
validate_numlist("resubmittedjobs", param, safe)
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
validate_num("limit", param, safe, optional=True)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
# possible combinations to check
# 1) taskname + status
# 2) taskname + status + failure
# 3) taskname + status + resubmitted + jobsetid
# 4) taskname + status == (1)
# 5) status + limit + getstatus + workername
# 6) taskname + runs + lumis
elif method in ['GET']:
validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("limit", param, safe, optional=True)
validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
validate_num("subjobdef", param, safe, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=True)
# possible combinations to check
# 1) workername + getstatus + limit
# 2) subresource + subjobdef + subuser
elif method in ['DELETE']:
pass
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
#authz_login_valid()
if method in ['PUT']:
validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
# TODO this should be changed to be non-optional
validate_str("generator", param, safe, RX_GENERATOR, optional=True)
validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_num("nonprodsw", param, safe, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
jobtype = safe.kwargs.get('jobtype', None)
validate_num("useparent", param, safe, optional=True)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_num("totalunits", param, safe, optional=True)
validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
validate_str("lfn", param, safe, RX_LFN, optional=True)
self._checkOutLFN(safe.kwargs)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
validate_strlist("userfiles", param, safe, RX_USERFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_num("saveoutput", param, safe, optional=True)
validate_num("faillimit", param, safe, optional=True)
validate_num("ignorelocality", param, safe, optional=True)
if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_num("publication", param, safe, optional=False)
validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication'])))
## Validations needed in case publication is on.
if safe.kwargs['publication']:
## The (user specified part of the) publication dataset name must be
## specified and must pass DBS validation. Since this is the correct
## validation function, it must be done before the
## validate_str("workflow", ...) and validate_str("publishname", ...)
## we have below. Not sure why RX_PUBLISH was introduced in CRAB, but
## it is not the correct regular expression for publication dataset
## name validation.
self._checkPublishDataName(param.kwargs, safe.kwargs['lfn'])
## 'publishname' was already validated above in _checkPublishDataName().
## But I am not sure if we can skip the validate_str('publishname', ...)
## or we need it so that all parameters are moved from param to safe.
## Therefore, I didn't remove the validate_str('publishname', ...),
## but only changed RX_PUBLISH -> RX_ANYTHING.
validate_str('publishname', param, safe, RX_ANYTHING, optional=False)
## If this is a MC generation job type, the primary dataset name will
## be used to define the primary dataset field in the publication
## dataset name. Therefore, the primary dataset name must pass DBS
## validation. Since in a MC generation job type the primary dataset
## name is saved in 'inputdata', _checkPrimaryDataset() actually
## validates 'inputdata'. Therefore, this function must come before
## the validate_str('inputdata', ...) we have below. OTOH, matching
## 'inputdata' agains RX_DATASET is not the correct validation for
## the primary dataset name.
if jobtype == 'PrivateMC':
self._checkPrimaryDataset(param.kwargs)
else:
## Not sure if we need to match publishname against RX_PUBLISH...
validate_str("publishname", param, safe, RX_PUBLISH, optional=False)
validate_num("publishgroupname", param, safe, optional=True)
## This line must come after _checkPublishDataName()
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
if jobtype == 'Analysis':
validate_str("inputdata", param, safe, RX_DATASET, optional=False)
## Added this if, because in this case 'inputdata' was already validated
## above in _checkPrimaryDataset(), and I am not sure if we can skip the
## validate_str('inputdata', ...) or we need it so that all parameters
## are moved from param to safe.
elif jobtype == 'PrivateMC' and safe.kwargs['publication']:
validate_str("inputdata", param, safe, RX_ANYTHING, optional=False)
else:
validate_str("inputdata", param, safe, RX_DATASET, optional=True)
validate_num("nonvaliddata", param, safe, optional=True)
#if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkASODestination(safe.kwargs['asyncdest'])
# We no longer use this attribute, but keep it around for older client compatibility
validate_num("blacklistT1", param, safe, optional=True)
validate_num("oneEventMode", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
#validate_str("scheduler", param, safe, RX_SCHEDULER)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
validate_str("asourl", param, safe, RX_ASOURL, optional=True)
validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
validate_str("collector", param, safe, RX_COLLECTOR, optional=True)
validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
validate_num("dryrun", param, safe, optional=True)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True)
## In a resubmission, the site black- and whitelists need to be interpreted
## differently than in an initial task submission. If there is no site black-
## or whitelist, set it to None and DataWorkflow will use the corresponding
## list defined in the initial task submission. If the site black- or whitelist
## is equal to the string 'empty', set it to an empty list and don't call
## validate_strlist as it would fail.
if 'siteblacklist' not in param.kwargs:
safe.kwargs['siteblacklist'] = None
elif param.kwargs['siteblacklist'] == 'empty':
safe.kwargs['siteblacklist'] = []
del param.kwargs['siteblacklist']
else:
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
if 'sitewhitelist' not in param.kwargs:
safe.kwargs['sitewhitelist'] = None
elif param.kwargs['sitewhitelist'] == 'empty':
safe.kwargs['sitewhitelist'] = []
del param.kwargs['sitewhitelist']
else:
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_numlist('jobids', param, safe)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_num("force", param, safe, optional=True)
elif method in ['GET']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric
## Used to determine how much information to return to the client for status.
## also used by report to determine if it has to check job states
validate_num("verbose", param, safe, optional=True)
## used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
## used by errors and report (short format in report means we do not query DBS)
validate_num('shortformat', param, safe, optional=True)
## validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_num("force", param, safe, optional=True)
validate_numlist('jobids', param, safe)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(
safe.kwargs["outfilelumis"]):
raise InvalidParameter(
"The number of runs and the number of lumis lists are different"
)
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_str("jobid", param, safe, RX_JOBID, optional=True)
#TODO: for backward compatibility. Get rid of the pandajobid once all jobs using it are done (one month after this gets released)
#(The following five lines can be deleted)
validate_num("pandajobid", param, safe, optional=True)
if bool(safe.kwargs["jobid"]) == bool(safe.kwargs["pandajobid"]):
raise InvalidParameter(
"Only one among jobid and pandajobid should be set")
#Oracle/cx_oracle/python stack does not like None for numbers, even if they are nullable
if safe.kwargs["pandajobid"] == None:
safe.kwargs["pandajobid"] = 0
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname",
param,
safe,
RX_PUBLISH,
optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumcksum",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumadler32",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("outlocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("outtmplocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("acquisitionera",
param,
safe,
RX_TASKNAME,
optional=False) #TODO Do we really need this?
validate_str(
"outdatasetname", param, safe, RX_OUTDSLFN,
optional=False) #TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
validate_num("events", param, safe, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
validate_num("directstageout", param, safe, optional=True)
safe.kwargs["directstageout"] = 'T' if safe.kwargs[
"directstageout"] else 'F' #'F' if not provided
elif method in ['POST']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_str("filestate",
param,
safe,
RX_FILESTATE,
optional=False)
elif method in ['GET']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
validate_num("howmany", param, safe, optional=True)
validate_strlist("lfn", param, safe, RX_LFN)
elif method in ['DELETE']:
authz_operator()
validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
validate_str("hours", param, safe, RX_HOURS, optional=True)
if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
" will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
if method == 'GET':
# Check for `performance` endpoint, documented in
# https://github.com/knly/WMArchiveAggregation
if len(param.args) == 1 and param.args[0] == 'performance':
safe.args.append(param.args[0])
param.args.remove(param.args[0])
# Validate arguments
validate_strlist('metrics[]', param, safe,
re.compile(r'^[a-zA-Z.]+'))
validate_strlist('axes[]', param, safe,
re.compile(r'^[a-zA-Z_]+'))
validate_strlist('suggestions[]', param, safe,
re.compile(r'^[a-zA-Z]+'))
date_pattern = PAT_YYYYMMDD
validate_str('start_date',
param,
safe,
date_pattern,
optional=True)
validate_str('end_date',
param,
safe,
date_pattern,
optional=True)
validate_rx('workflow', param, safe, optional=True)
validate_rx('task', param, safe, optional=True)
validate_rx('host', param, safe, optional=True)
validate_rx('site', param, safe, optional=True)
validate_rx('jobtype', param, safe, optional=True)
validate_rx('jobstate', param, safe, optional=True)
validate_rx('acquisitionEra', param, safe, optional=True)
validate_rx('exitCode', param, safe, optional=True)
validate_rx('exitStep', param, safe, optional=True)
validate_str('_', param, safe, PAT_INFO, optional=True)
return True
if 'query' in param.kwargs.keys():
validate_str('query', param, safe, PAT_QUERY, optional=True)
for key in ['status', 'jobs', '_']:
if key in param.kwargs.keys():
validate_str(key, param, safe, PAT_INFO, optional=True)
# test if user provided uid
if len(param.args) == 1 and PAT_UID.match(param.args[0]):
safe.args.append(param.args[0])
param.args.remove(param.args[0])
return True
elif method == 'POST':
if not param.args or not param.kwargs:
return False # this class does not need any parameters
return True
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid(
) #TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here.
# Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource)
if method in ['PUT']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_num("subjobdef", param, safe, optional=True)
validate_str("substatus", param, safe, RX_STATUS, optional=False)
validate_strlist("subblocks", param, safe, RX_BLOCK)
validate_str("subfailure",
param,
safe,
RX_TEXT_FAIL,
optional=True)
validate_str("subuser", param, safe, RX_DN, optional=False)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
validate_str("status", param, safe, RX_STATUS, optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("jobset", param, safe, optional=True)
validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
validate_numlist("resubmittedjobs", param, safe)
validate_str("workername",
param,
safe,
RX_WORKER_NAME,
optional=True)
validate_str("subresource",
param,
safe,
RX_SUBPOSTWORKER,
optional=True)
validate_num("limit", param, safe, optional=True)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
# possible combinations to check
# 1) taskname + status
# 2) taskname + status + failure
# 3) taskname + status + resubmitted + jobsetid
# 4) taskname + status == (1)
# 5) status + limit + getstatus + workername
# 6) taskname + runs + lumis
elif method in ['GET']:
validate_str("workername",
param,
safe,
RX_WORKER_NAME,
optional=True)
validate_str("getstatus", param, safe, RX_STATUS, optional=True)
validate_num("limit", param, safe, optional=True)
validate_str("subresource",
param,
safe,
RX_SUBGETWORKER,
optional=True)
validate_num("subjobdef", param, safe, optional=True)
validate_str("subuser", param, safe, RX_DN, optional=True)
# possible combinations to check
# 1) workername + getstatus + limit
# 2) subresource + subjobdef + subuser
elif method in ['DELETE']:
pass
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
#authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
validate_str("generator", param, safe, RX_GENERATOR, optional=True)
validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_num("nonprodsw", param, safe, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
jobtype = safe.kwargs.get('jobtype', None)
if jobtype == 'Analysis':
validate_str("inputdata", param, safe, RX_DATASET, optional=False)
else:
validate_str("inputdata", param, safe, RX_DATASET, optional=True)
validate_num("useparent", param, safe, optional=True)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_num("totalunits", param, safe, optional=True)
validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
validate_str("lfnprefix", param, safe, RX_LFNPATH, optional=True) #keeping this for compatibility with old clients. Will be converted into a full lfn
validate_str("lfn", param, safe, RX_LFN, optional=True)
self._checkOutLFN(safe.kwargs)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
validate_strlist("userfiles", param, safe, RX_USERFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_num("saveoutput", param, safe, optional=True)
validate_num("faillimit", param, safe, optional=True)
validate_num("ignorelocality", param, safe, optional=True)
if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_str("publishname", param, safe, RX_PUBLISH, optional=False)
validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=True)
validate_num("publication", param, safe, optional=False)
#if publication is set as true both publishDataName and publishDbsUrl are needed
if safe.kwargs["publication"] and not (bool(safe.kwargs["publishname"]) and bool(safe.kwargs["publishdbsurl"])):
raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication")
#if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkASODestination(safe.kwargs['asyncdest'])
# We no longer use this attribute, but keep it around for older client compatibility
validate_num("blacklistT1", param, safe, optional=True)
validate_num("oneEventMode", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
#validate_str("scheduler", param, safe, RX_SCHEDULER)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
validate_str("asourl", param, safe, RX_ASOURL, optional=True)
validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_numlist('jobids', param, safe)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
elif method in ['GET']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric
## Used to determine how much information to return to the client for status.
## also used by report to determine if it has to check job states
validate_num("verbose", param, safe, optional=True)
## used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
## used by errors and report (short format in report means we do not query DBS)
validate_num('shortformat', param, safe, optional=True)
## validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_num("force", param, safe, optional=True)
validate_numlist('jobids', param, safe)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
jobtype = safe.kwargs.get('jobtype', None)
if jobtype == 'Analysis':
validate_str("inputdata",
param,
safe,
RX_DATASET,
optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(
safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(
safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_str("cachefilename",
param,
safe,
RX_CACHENAME,
optional=False)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
#validate_str("userisburl", param, safe, re.compile(r"^[A-Za-z]*$"), optional=False)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_str("publishname", param, safe, RX_PUBLISH, optional=True)
validate_str("publishdbsurl",
param,
safe,
RX_DBSURL,
optional=True)
validate_num("publication", param, safe, optional=False)
#if publication is set as true both publishDataName and publishDbsUrl are needed
if safe.kwargs["publication"] and not (
bool(safe.kwargs["publishname"])
and bool(safe.kwargs["publishdbsurl"])):
raise InvalidParameter(
"You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication"
)
#if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
if bool(safe.kwargs["publishname"]) != bool(
safe.kwargs["publishdbsurl"]):
raise InvalidParameter(
"You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication"
)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkSite(safe.kwargs['asyncdest'])
validate_num("blacklistT1", param, safe, optional=False)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=True)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter(
"The number of runs and the number of lumis lists are different"
)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(
safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(
safe.kwargs['sitewhitelist'])
elif method in ['GET']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
validate_str('subresource',
param,
safe,
RX_SUBRESTAT,
optional=True)
#parameters of subresources calls has to be put here
#used by get latest
validate_num('age', param, safe, optional=True)
#used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
#used by errors
validate_num('shortformat', param, safe, optional=True)
#validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in [
'data', 'logs'
] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter(
"You need to specify the number of jobs to retrieve or their ids."
)
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_num("force", param, safe, optional=True)
def validate(self, apiobj, method, api, param, safe): #pylint: disable=unused-argument
"""Validating all the input parameter as enforced by the WMCore.REST module"""
if method in ['PUT']:
username = cherrypy.request.user['login'] # username registered in CMS WEB frontend
requestname = param.kwargs['workflow']
param.kwargs['workflow'] = generateTaskName(username, requestname)
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
# TODO this should be changed to be non-optional
validate_str("generator", param, safe, RX_GENERATOR, optional=True)
validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_num("nonprodsw", param, safe, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
validate_num("useparent", param, safe, optional=True)
validate_str("secondarydata", param, safe, RX_DATASET, optional=True)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
try:
validate_num("totalunits", param, safe, optional=True)
except InvalidParameter:
validate_real("totalunits", param, safe, optional=True)
validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
validate_str("debugfilename", param, safe, RX_CACHENAME, optional=True)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
validate_str("lfn", param, safe, RX_LFN, optional=True)
self._checkOutLFN(safe.kwargs, username)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE, custom_err="Incorrect 'JobType.outputFiles' parameter. " \
"Allowed regexp for each filename: '%s'." % RX_ADDFILE.pattern)
validate_strlist("userfiles", param, safe, RX_USERFILE, custom_err="Incorrect 'Data.userInputFiles' parameter. " \
"Allowed regexp for each filename: '%s'." % RX_USERFILE.pattern)
validate_num("savelogsflag", param, safe, optional=False)
validate_num("saveoutput", param, safe, optional=True)
validate_num("faillimit", param, safe, optional=True)
validate_num("ignorelocality", param, safe, optional=True)
if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_num("publication", param, safe, optional=False)
validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication'])))
## We might want to remove publishname once the backward compatibility
## wont be needed anymore. Then we can just keep publishname2
## NB: AFAIK the only client not using the CRABLibrary but direct cutl is HC,
## therefore we will need to make sure we do not break it!
## The following two lines will be removed in the future once we will
## not need backward compatibility anymore
self._checkPublishDataName(param.kwargs, safe.kwargs['lfn'], requestname, username)
validate_str('publishname', param, safe, RX_ANYTHING, optional=True)
##And this if as well, just do self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
if not safe.kwargs["publishname"]: #new clients won't define this anymore
## The (user specified part of the) publication dataset name must be
## specified and must pass DBS validation. Since this is the correct
## validation function, it must be done before the
## validate_str("publishname", ...) we have below.
self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
else:
param.kwargs["publishname2"] = safe.kwargs["publishname"]
## 'publishname' was already validated above in _checkPublishDataName().
## Calling validate_str with a fake regexp to move the param to the
## list of validated inputs
validate_str("publishname2", param, safe, RX_ANYTHING, optional=True)
validate_num("publishgroupname", param, safe, optional=True)
if safe.kwargs['jobtype'] == 'PrivateMC':
if param.kwargs['inputdata']:
msg = "Invalid 'inputdata' parameter."
msg += " Job type PrivateMC does not take any input dataset."
msg += " If you really intend to run over an input dataset, then you must use job type Analysis."
raise InvalidParameter(msg)
if safe.kwargs['userfiles']:
msg = "Invalid 'userfiles' parameter."
msg += " Job type PrivateMC does not take any input files."
msg += " If you really intend to run over input files, then you must use job type Analysis."
raise InvalidParameter(msg)
## Client versions < 3.3.1511 may put in the input dataset something that is not
## really an input dataset (for PrivateMC or user input files). So the only case
## in which we are sure that we have to validate the input dataset is when the
## workflow type is Analysis, the workflow does not run on user input files and
## an input dataset is defined (scriptExe may not define an input).
## Once we don't care anymore about backward compatibility with client < 3.3.1511,
## we can uncomment the 1st line below and delete the next 4 lines.
#validate_str("inputdata", param, safe, RX_DATASET, optional=True)
if safe.kwargs['jobtype'] == 'Analysis' and not safe.kwargs['userfiles'] and 'inputdata' in param.kwargs:
validate_str("inputdata", param, safe, RX_DATASET, optional=True)
else:
validate_str("inputdata", param, safe, RX_ANYTHING, optional=True)
## The client is not forced to define the primary dataset. So make sure to have
## defaults or take it from the input dataset. The primary dataset is needed for
## the LFN of the output/log files and for publication. We want to have it well
## defined even if publication and/or transfer to storage are off.
if safe.kwargs['inputdata']:
param.kwargs['primarydataset'] = safe.kwargs['inputdata'].split('/')[1]
if not param.kwargs.get('primarydataset', None):
if safe.kwargs['jobtype'] == 'PrivateMC':
param.kwargs['primarydataset'] = "CRAB_PrivateMC"
elif safe.kwargs['jobtype'] == 'Analysis' and safe.kwargs['userfiles']:
param.kwargs['primarydataset'] = "CRAB_UserFiles"
else:
param.kwargs['primarydataset'] = "CRAB_NoInput"
## We validate the primary dataset agains DBS rules even if publication is off,
## because in the future we may want to give the possibility to users to publish
## a posteriori.
self._checkPrimaryDataset(param.kwargs, optional=False)
validate_str("primarydataset", param, safe, RX_LFNPRIMDS, optional=False)
validate_num("nonvaliddata", param, safe, optional=True)
#if one and only one between outputDatasetTag and publishDbsUrl is set raise an error (we need both or none of them)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkASODestination(safe.kwargs['asyncdest'])
# We no longer use this attribute, but keep it around for older client compatibility
validate_num("blacklistT1", param, safe, optional=True)
validate_num("oneEventMode", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect tfileoutfiles parameter (TFileService). " \
"Allowed regexp: '%s'." % RX_OUTFILES.pattern)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect edmoutfiles parameter (PoolOutputModule). " \
"Allowed regexp: '%s'." % RX_OUTFILES.pattern)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
validate_str("asourl", param, safe, RX_ASOURL, optional=True)
validate_str("asodb", param, safe, RX_ASODB, optional=True)
safe.kwargs["asourl"], safe.kwargs["asodb"] = self._getAsoConfig(safe.kwargs["asourl"], safe.kwargs["asodb"])
validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
validate_str("collector", param, safe, RX_COLLECTOR, optional=True)
validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
validate_num("dryrun", param, safe, optional=True)
validate_num("ignoreglobalblacklist", param, safe, optional=True)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True)
validate_strlist('jobids', param, safe, RX_JOBID)
## In a resubmission, the site black- and whitelists need to be interpreted
## differently than in an initial task submission. If there is no site black-
## or whitelist, set it to None and DataWorkflow will use the corresponding
## list defined in the initial task submission. If the site black- or whitelist
## is equal to the string 'empty', set it to an empty list and don't call
## validate_strlist as it would fail.
if 'siteblacklist' not in param.kwargs:
safe.kwargs['siteblacklist'] = None
elif param.kwargs['siteblacklist'] == 'empty':
safe.kwargs['siteblacklist'] = []
del param.kwargs['siteblacklist']
else:
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
if 'sitewhitelist' not in param.kwargs:
safe.kwargs['sitewhitelist'] = None
elif param.kwargs['sitewhitelist'] == 'empty':
safe.kwargs['sitewhitelist'] = []
del param.kwargs['sitewhitelist']
else:
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("force", param, safe, optional=True)
validate_num("publication", param, safe, optional=True)
elif method in ['GET']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric
## Used to determine how much information to return to the client for status.
## also used by report to determine if it has to check job states
validate_num("verbose", param, safe, optional=True)
## used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_strlist('jobids', param, safe, RX_JOBID)
## used by errors and report (short format in report means we do not query DBS)
validate_num('shortformat', param, safe, optional=True)
# used by publicationStatus
validate_str("asourl", param, safe, RX_ASOURL, optional=True)
validate_str("asodb", param, safe, RX_ASODB, optional=True)
## validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_num("force", param, safe, optional=True)
validate_str("killwarning", param, safe, RX_TEXT_FAIL, optional=True)
#decode killwarning message if present
if safe.kwargs['killwarning']:
try:
safe.kwargs['killwarning'] = b64decode(safe.kwargs['killwarning'])
except TypeError:
raise InvalidParameter("Failure message is not in the accepted format")
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
#authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
validate_str("generator", param, safe, RX_GENERATOR, optional=True)
validate_str("eventsperlumi",
param,
safe,
RX_LUMIEVENTS,
optional=True)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_num("nonprodsw", param, safe, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
if not safe.kwargs[
"nonprodsw"]: #if the user wants to allow non-production releases
self._checkReleases(safe.kwargs['jobarch'],
safe.kwargs['jobsw'])
jobtype = safe.kwargs.get('jobtype', None)
if jobtype == 'Analysis':
validate_str("inputdata",
param,
safe,
RX_DATASET,
optional=False)
else:
validate_str("inputdata",
param,
safe,
RX_DATASET,
optional=True)
validate_num("useparent", param, safe, optional=True)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(
safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(
safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_num("totalunits", param, safe, optional=True)
validate_str("cachefilename",
param,
safe,
RX_CACHENAME,
optional=False)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
validate_str(
"lfnprefix", param, safe, RX_LFNPATH, optional=True
) #keeping this for compatibility with old clients. Will be converted into a full lfn
validate_str("lfn", param, safe, RX_LFN, optional=True)
self._checkOutLFN(safe.kwargs)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
validate_strlist("userfiles", param, safe, RX_USERFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_num("saveoutput", param, safe, optional=True)
validate_num("faillimit", param, safe, optional=True)
validate_num("ignorelocality", param, safe, optional=True)
if safe.kwargs[
'ignorelocality'] and self.centralcfg.centralconfig.get(
'ign-locality-blacklist', []):
safe.kwargs['siteblacklist'] += self._expandSites(
self.centralcfg.centralconfig['ign-locality-blacklist'])
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_str("publishname",
param,
safe,
RX_PUBLISH,
optional=False)
validate_str("publishdbsurl",
param,
safe,
RX_DBSURL,
optional=True)
validate_num("publication", param, safe, optional=False)
#if publication is set as true both publishDataName and publishDbsUrl are needed
if safe.kwargs["publication"] and not (
bool(safe.kwargs["publishname"])
and bool(safe.kwargs["publishdbsurl"])):
raise InvalidParameter(
"You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication"
)
#if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkASODestination(safe.kwargs['asyncdest'])
# We no longer use this attribute, but keep it around for older client compatibility
validate_num("blacklistT1", param, safe, optional=True)
validate_num("oneEventMode", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
#validate_str("scheduler", param, safe, RX_SCHEDULER)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter(
"The number of runs and the number of lumis lists are different"
)
validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
validate_str("asourl", param, safe, RX_ASOURL, optional=True)
validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
validate_str("scheddname",
param,
safe,
RX_SCHEDD_NAME,
optional=True)
validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(
safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(
safe.kwargs['sitewhitelist'])
validate_numlist('jobids', param, safe)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
elif method in ['GET']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
validate_str('subresource',
param,
safe,
RX_SUBRESTAT,
optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
validate_str('timestamp', param, safe, RX_DATE,
optional=True) ## inserted by eric
## Used to determine how much information to return to the client for status.
## also used by report to determine if it has to check job states
validate_num("verbose", param, safe, optional=True)
## used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
## used by errors and report (short format in report means we do not query DBS)
validate_num('shortformat', param, safe, optional=True)
## validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in [
'data', 'logs'
] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter(
"You need to specify the number of jobs to retrieve or their ids."
)
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_num("force", param, safe, optional=True)
validate_numlist('jobids', param, safe)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(
safe.kwargs["outfilelumis"]):
raise InvalidParameter(
"The number of runs and the number of lumis lists are different"
)
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_num("pandajobid", param, safe, optional=False)
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname",
param,
safe,
RX_PUBLISH,
optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumcksum",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumadler32",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("outlocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("outtmplocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("acquisitionera",
param,
safe,
RX_TASKNAME,
optional=False) #TODO Do we really need this?
validate_str(
"outdatasetname", param, safe, RX_OUTDSLFN,
optional=False) #TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
validate_num("events", param, safe, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
validate_num("directstageout", param, safe, optional=True)
safe.kwargs["directstageout"] = 'T' if safe.kwargs[
"directstageout"] else 'F' #'F' if not provided
elif method in ['POST']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_str("filestate",
param,
safe,
RX_FILESTATE,
optional=False)
elif method in ['GET']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
elif method in ['DELETE']:
authz_operator()
validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
validate_str("hours", param, safe, RX_HOURS, optional=True)
if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
" will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
jobtype = safe.kwargs.get('jobtype', None)
if jobtype == 'Analysis':
validate_str("inputdata", param, safe, RX_DATASET, optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
#validate_str("userisburl", param, safe, re.compile(r"^[A-Za-z]*$"), optional=False)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_str("publishname", param, safe, RX_PUBLISH, optional=True)
validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=True)
validate_num("publication", param, safe, optional=False)
#if publication is set as true both publishDataName and publishDbsUrl are needed
if safe.kwargs["publication"] and not (bool(safe.kwargs["publishname"]) and bool(safe.kwargs["publishdbsurl"])):
raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication")
#if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
if bool(safe.kwargs["publishname"]) != bool(safe.kwargs["publishdbsurl"]):
raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication")
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkSite(safe.kwargs['asyncdest'])
validate_num("blacklistT1", param, safe, optional=False)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=True)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
elif method in ['POST']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
elif method in ['GET']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
#parameters of subresources calls has to be put here
#used by get latest
validate_num('age', param, safe, optional=True)
#used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
#used by errors
validate_num('shortformat', param, safe, optional=True)
#validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
validate_num("force", param, safe, optional=True)
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_strlist("outfileruns", param, safe, RX_RUNS)
if len(safe.kwargs["outfileruns"]) != len(
safe.kwargs["outfilelumis"]):
raise InvalidParameter(
"The number of runs and the number of lumis lists are different"
)
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
# inparentlfns will be inserted in Oracle as CLOB, so it must be a string
safe.kwargs['inparentlfns'] = str(safe.kwargs['inparentlfns'])
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_str("jobid", param, safe, RX_JOBID, optional=True)
safe.kwargs["pandajobid"] = 0
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname",
param,
safe,
RX_PUBLISH,
optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_num("checksumcksum", param, safe, optional=False)
validate_str("checksumadler32",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("outlocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("outtmplocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("acquisitionera",
param,
safe,
RX_TASKNAME,
optional=False)
validate_str("outdatasetname",
param,
safe,
RX_OUTDSLFN,
optional=False)
# need to use RX_PARENTLFN becasue same API is also used for input metadata
validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
validate_str("outtmplfn", param, safe, RX_LFN, optional=True)
validate_num("events", param, safe, optional=False)
validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
validate_num("directstageout", param, safe, optional=True)
safe.kwargs["directstageout"] = 'T' if safe.kwargs[
"directstageout"] else 'F' #'F' if not provided
elif method in ['POST']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_str("filestate",
param,
safe,
RX_FILESTATE,
optional=False)
elif method in ['GET']:
validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
validate_num("howmany", param, safe, optional=True)
validate_strlist("lfn", param, safe, RX_LFN)
elif method in ['DELETE']:
authz_operator()
validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
validate_str("hours", param, safe, RX_HOURS, optional=True)
if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
" will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
if method in ['PUT']:
username = cherrypy.request.user['login'] # username registered in SiteDB
requestname = param.kwargs['workflow']
param.kwargs['workflow'] = generateTaskName(username, requestname)
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
# TODO this should be changed to be non-optional
validate_str("generator", param, safe, RX_GENERATOR, optional=True)
validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
validate_num("nonprodsw", param, safe, optional=False)
validate_str("jobarch", param, safe, RX_ARCH, optional=False)
if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
validate_num("useparent", param, safe, optional=True)
validate_str("secondarydata", param, safe, RX_DATASET, optional=True)
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
validate_num("algoargs", param, safe, optional=False)
validate_num("totalunits", param, safe, optional=True)
validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
validate_str("debugfilename", param, safe, RX_CACHENAME, optional=True)
validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
validate_str("lfn", param, safe, RX_LFN, optional=True)
self._checkOutLFN(safe.kwargs, username)
validate_strlist("addoutputfiles", param, safe, RX_ADDFILE, custom_err="Incorrect 'JobType.outputFiles' parameter. " \
"Allowed regexp: '%s'." % RX_ADDFILE.pattern)
validate_strlist("userfiles", param, safe, RX_USERFILE)
validate_num("savelogsflag", param, safe, optional=False)
validate_num("saveoutput", param, safe, optional=True)
validate_num("faillimit", param, safe, optional=True)
validate_num("ignorelocality", param, safe, optional=True)
if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
validate_num("publication", param, safe, optional=False)
validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication'])))
## We might want to remove publishname once the backward compatibility
## wont be needed anymore. Then we can just keep publishname2
## NB: AFAIK the only client not using the CRABLibrary but direct cutl is HC,
## therefore we will need to make sure we do not break it!
## The following two lines will be removed in the future once we will
## not need backward compatibility anymore
self._checkPublishDataName(param.kwargs, safe.kwargs['lfn'], requestname, username)
validate_str('publishname', param, safe, RX_ANYTHING, optional=True)
##And this if as well, just do self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
if not safe.kwargs["publishname"]: #new clients won't define this anymore
## The (user specified part of the) publication dataset name must be
## specified and must pass DBS validation. Since this is the correct
## validation function, it must be done before the
## validate_str("publishname", ...) we have below.
self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
else:
param.kwargs["publishname2"] = safe.kwargs["publishname"]
## 'publishname' was already validated above in _checkPublishDataName().
## Calling validate_str with a fake regexp to move the param to the
## list of validated inputs
validate_str("publishname2", param, safe, RX_ANYTHING, optional=True)
validate_num("publishgroupname", param, safe, optional=True)
if safe.kwargs['jobtype'] == 'PrivateMC':
if param.kwargs['inputdata']:
msg = "Invalid 'inputdata' parameter."
msg += " Job type PrivateMC does not take any input dataset."
msg += " If you really intend to run over an input dataset, then you must use job type Analysis."
raise InvalidParameter(msg)
if safe.kwargs['userfiles']:
msg = "Invalid 'userfiles' parameter."
msg += " Job type PrivateMC does not take any input files."
msg += " If you really intend to run over input files, then you must use job type Analysis."
raise InvalidParameter(msg)
## Client versions < 3.3.1511 may put in the input dataset something that is not
## really an input dataset (for PrivateMC or user input files). So the only case
## in which we are sure that we have to validate the input dataset is when the
## workflow type is Analysis, the workflow does not run on user input files and
## an input dataset is defined (scriptExe may not define an input).
## Once we don't care anymore about backward compatibility with client < 3.3.1511,
## we can uncomment the 1st line below and delete the next 4 lines.
#validate_str("inputdata", param, safe, RX_DATASET, optional=True)
if safe.kwargs['jobtype'] == 'Analysis' and not safe.kwargs['userfiles'] and 'inputdata' in param.kwargs:
validate_str("inputdata", param, safe, RX_DATASET, optional=True)
else:
validate_str("inputdata", param, safe, RX_ANYTHING, optional=True)
## The client is not forced to define the primary dataset. So make sure to have
## defaults or take it from the input dataset. The primary dataset is needed for
## the LFN of the output/log files and for publication. We want to have it well
## defined even if publication and/or transfer to storage are off.
if safe.kwargs['inputdata']:
param.kwargs['primarydataset'] = safe.kwargs['inputdata'].split('/')[1]
if not param.kwargs.get('primarydataset', None):
if safe.kwargs['jobtype'] == 'PrivateMC':
param.kwargs['primarydataset'] = "CRAB_PrivateMC"
elif safe.kwargs['jobtype'] == 'Analysis' and safe.kwargs['userfiles']:
param.kwargs['primarydataset'] = "CRAB_UserFiles"
else:
param.kwargs['primarydataset'] = "CRAB_NoInput"
## We validate the primary dataset agains DBS rules even if publication is off,
## because in the future we may want to give the possibility to users to publish
## a posteriori.
self._checkPrimaryDataset(param.kwargs, optional=False)
validate_str("primarydataset", param, safe, RX_LFNPRIMDS, optional=False)
validate_num("nonvaliddata", param, safe, optional=True)
#if one and only one between outputDatasetTag and publishDbsUrl is set raise an error (we need both or none of them)
validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
self._checkASODestination(safe.kwargs['asyncdest'])
# We no longer use this attribute, but keep it around for older client compatibility
validate_num("blacklistT1", param, safe, optional=True)
validate_num("oneEventMode", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect tfileoutfiles parameter (TFileService). " \
"Allowed regexp: '%s'." % RX_OUTFILES.pattern)
validate_strlist("edmoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect edmoutfiles parameter (PoolOutputModule). " \
"Allowed regexp: '%s'." % RX_OUTFILES.pattern)
validate_strlist("runs", param, safe, RX_RUNS)
validate_strlist("lumis", param, safe, RX_LUMIRANGE)
#validate_str("scheduler", param, safe, RX_SCHEDULER)
if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
raise InvalidParameter("The number of runs and the number of lumis lists are different")
validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
validate_str("asourl", param, safe, RX_ASOURL, optional=True)
validate_str("asodb", param, safe, RX_ASODB, optional=True)
safe.kwargs["asourl"], safe.kwargs["asodb"] = self._getAsoConfig(safe.kwargs["asourl"], safe.kwargs["asodb"])
validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
validate_str("collector", param, safe, RX_COLLECTOR, optional=True)
validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
validate_num("dryrun", param, safe, optional=True)
validate_num("ignoreglobalblacklist", param, safe, optional=True)
elif method in ['POST']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True)
validate_numlist('jobids', param, safe)
## In a resubmission, the site black- and whitelists need to be interpreted
## differently than in an initial task submission. If there is no site black-
## or whitelist, set it to None and DataWorkflow will use the corresponding
## list defined in the initial task submission. If the site black- or whitelist
## is equal to the string 'empty', set it to an empty list and don't call
## validate_strlist as it would fail.
if 'siteblacklist' not in param.kwargs:
safe.kwargs['siteblacklist'] = None
elif param.kwargs['siteblacklist'] == 'empty':
safe.kwargs['siteblacklist'] = []
del param.kwargs['siteblacklist']
else:
validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
if 'sitewhitelist' not in param.kwargs:
safe.kwargs['sitewhitelist'] = None
elif param.kwargs['sitewhitelist'] == 'empty':
safe.kwargs['sitewhitelist'] = []
del param.kwargs['sitewhitelist']
else:
validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
validate_num("maxjobruntime", param, safe, optional=True)
validate_num("maxmemory", param, safe, optional=True)
validate_num("numcores", param, safe, optional=True)
validate_num("priority", param, safe, optional=True)
validate_num("force", param, safe, optional=True)
validate_num("publication", param, safe, optional=True)
elif method in ['GET']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
validate_str('username', param, safe, RX_USERNAME, optional=True)
validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric
## Used to determine how much information to return to the client for status.
## also used by report to determine if it has to check job states
validate_num("verbose", param, safe, optional=True)
## used by get log, get data
validate_num('limit', param, safe, optional=True)
validate_num('exitcode', param, safe, optional=True)
validate_numlist('jobids', param, safe)
## used by errors and report (short format in report means we do not query DBS)
validate_num('shortformat', param, safe, optional=True)
## validation parameters
if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
raise InvalidParameter("Invalid input parameters")
if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")
elif method in ['DELETE']:
validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
validate_num("force", param, safe, optional=True)
validate_numlist('jobids', param, safe)
validate_str("killwarning", param, safe, RX_TEXT_FAIL, optional=True)
#decode killwarning message if present
if safe.kwargs['killwarning']:
try:
safe.kwargs['killwarning'] = b64decode(safe.kwargs['killwarning'])
except TypeError:
raise InvalidParameter("Failure message is not in the accepted format")
def validate(self, apiobj, method, api, param, safe):
"""Validating all the input parameter as enforced by the WMCore.REST module"""
authz_login_valid()
if method in ['PUT']:
#TODO check optional parameter
#TODO check all the regexp
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
validate_numlist("outfileruns", param, safe)
if len(safe.kwargs["outfileruns"]) != len(
safe.kwargs["outfilelumis"]):
raise InvalidParameter(
"The number of runs and the number of lumis lists are different"
)
validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
validate_num("pandajobid", param, safe, optional=False)
validate_num("outsize", param, safe, optional=False)
validate_str("publishdataname",
param,
safe,
RX_PUBLISH,
optional=False)
validate_str("appver", param, safe, RX_CMSSW, optional=False)
validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
validate_str("checksummd5",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumcksum",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("checksumadler32",
param,
safe,
RX_CHECKSUM,
optional=False)
validate_str("outlocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("outtmplocation",
param,
safe,
RX_CMSSITE,
optional=False)
validate_str("acquisitionera",
param,
safe,
RX_WORKFLOW,
optional=False) #TODO Do we really need this?
validate_str(
"outdatasetname", param, safe, RX_OUTDSLFN,
optional=False) #TODO temporary, need to come up with a regex
validate_str("outlfn", param, safe, RX_LFN, optional=False)
validate_num("events", param, safe, optional=False)
elif method in ['POST']:
raise NotImplementedError
elif method in ['GET']:
validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
elif method in ['DELETE']:
raise NotImplementedError
