def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['POST']: validate_str('subresource', param, safe, RX_SUBRES_TASK, optional=False) validate_str("workflow", param, safe, RX_TASKNAME, optional=True) validate_str("warning", param, safe, RX_TEXT_FAIL, optional=True) validate_str("webdirurl", param, safe, RX_URL, optional=True) validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True) validate_strlist("outputdatasets", param, safe, RX_OUT_DATASET) validate_str("taskstatus", param, safe, RX_STATUS, optional=True) validate_num("ddmreqid", param, safe, optional=True) elif method in ['GET']: validate_str('subresource', param, safe, RX_SUBRES_TASK, optional=False) validate_str("workflow", param, safe, RX_TASKNAME, optional=True) validate_str('taskstatus', param, safe, RX_STATUS, optional=True) validate_str('username', param, safe, RX_USERNAME, optional=True) validate_str('minutes', param, safe, RX_RUNS, optional=True)
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['GET']: validate_str('subresource', param, safe, RX_SUBRES, optional=True) validate_str("hashkey", param, safe, RX_HASH, optional=True) validate_str("username", param, safe, RX_USERNAME, optional=True) validate_num("verbose", param, safe, optional=True)
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['GET']: validate_str('subresource', param, safe, RX_SUBRES, optional=True) validate_str("hashkey", param, safe, RX_HASH, optional=True) validate_num("verbose", param, safe, optional=True) validate_str("username", param, safe, RX_USERNAME, optional=True) if safe.kwargs['username']: authz_operator(safe.kwargs['username'])
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: #TODO check optional parameter #TODO check all the regexp validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_strlist("outfilelumis", param, safe, RX_LUMILIST) validate_numlist("outfileruns", param, safe) if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]): raise InvalidParameter("The number of runs and the number of lumis lists are different") validate_strlist("inparentlfns", param, safe, RX_PARENTLFN) validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True) validate_str("jobid", param, safe, RX_JOBID, optional=True) #TODO: for backward compatibility. Get rid of the pandajobid once all jobs using it are done (one month after this gets released) #(The following five lines can be deleted) validate_num("pandajobid", param, safe, optional=True) if bool(safe.kwargs["jobid"]) == bool(safe.kwargs["pandajobid"]): raise InvalidParameter("Only one among jobid and pandajobid should be set") #Oracle/cx_oracle/python stack does not like None for numbers, even if they are nullable if safe.kwargs["pandajobid"] == None: safe.kwargs["pandajobid"] = 0 validate_num("outsize", param, safe, optional=False) validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False) validate_str("appver", param, safe, RX_CMSSW, optional=False) validate_str("outtype", param, safe, RX_OUTTYPES, optional=False) validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False) validate_str("outlocation", param, safe, RX_CMSSITE, optional=False) validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False) validate_str("acquisitionera", param, safe, RX_TASKNAME, optional=False)#TODO Do we really need this? validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False) validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True) validate_num("events", param, safe, optional=False) validate_str("filestate", param, safe, RX_FILESTATE, optional=True) validate_num("directstageout", param, safe, optional=True) safe.kwargs["directstageout"] = 'T' if safe.kwargs["directstageout"] else 'F' #'F' if not provided elif method in ['POST']: validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_str("outlfn", param, safe, RX_LFN, optional=False) validate_str("filestate", param, safe, RX_FILESTATE, optional=False) elif method in ['GET']: validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_str("filetype", param, safe, RX_OUTTYPES, optional=False) validate_num("howmany", param, safe, optional=True) elif method in ['DELETE']: authz_operator() validate_str("taskname", param, safe, RX_TASKNAME, optional=True) validate_str("hours", param, safe, RX_HOURS, optional=True) if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]): raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\ " will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: validate_str("hashkey", param, safe, RX_HASH, optional=False) validate_num("newchecksum", param, safe, optional=True) validate_tarfile("inputfile", param, safe, 'hashkey', optional=False) if method in ['GET']: validate_str("hashkey", param, safe, RX_HASH, optional=False) validate_str("username", param, safe, RX_USERNAME, optional=True) if safe.kwargs['username']: authz_operator(safe.kwargs['username'])
def validate(self, apiobj, method, api, param, safe): validate_str("etag", param, safe, re.compile("^[a-z]*$"), optional=True) validate_num("lim", param, safe, bare=True, optional=True, minval=0, maxval=10)
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: validate_str("workflow", param, safe, RX_WORKFLOW, optional=False) validate_num("subjobdef", param, safe, optional=True) validate_str("substatus", param, safe, RX_STATUS, optional=False) validate_strlist("subblocks", param, safe, RX_BLOCK) validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True) validate_str("subuser", param, safe, RX_DN, optional=False) elif method in ['POST']: validate_str("workflow", param, safe, RX_WORKFLOW, optional=True) validate_str("status", param, safe, RX_STATUS, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_num("jobset", param, safe, optional=True) validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True) validate_numlist("resubmittedjobs", param, safe) validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True) validate_num("limit", param, safe, optional=True) validate_strlist("runs", param, safe, RX_RUNS) validate_strlist("lumis", param, safe, RX_LUMIRANGE) validate_strlist("outputdataset", param, safe, RX_OUT_DATASET) # possible combinations to check # 1) taskname + status # 2) taskname + status + failure # 3) taskname + status + resubmitted + jobsetid # 4) taskname + status == (1) # 5) status + limit + getstatus + workername # 6) taskname + runs + lumis elif method in ['GET']: validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_num("limit", param, safe, optional=True) validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True) validate_num("subjobdef", param, safe, optional=True) validate_str("subuser", param, safe, RX_DN, optional=True) # possible combinations to check # 1) workername + getstatus + limit # 2) subresource + subjobdef + subuser elif method in ['DELETE']: pass
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() #TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here. # Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource) if method in ['PUT']: validate_str("workflow", param, safe, RX_WORKFLOW, optional=False) validate_num("subjobdef", param, safe, optional=True) validate_str("substatus", param, safe, RX_STATUS, optional=False) validate_strlist("subblocks", param, safe, RX_BLOCK) validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True) validate_str("subuser", param, safe, RX_DN, optional=False) elif method in ['POST']: validate_str("workflow", param, safe, RX_WORKFLOW, optional=True) validate_str("status", param, safe, RX_STATUS, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_num("jobset", param, safe, optional=True) validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True) validate_numlist("resubmittedjobs", param, safe) validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True) validate_num("limit", param, safe, optional=True) validate_strlist("runs", param, safe, RX_RUNS) validate_strlist("lumis", param, safe, RX_LUMIRANGE) # possible combinations to check # 1) taskname + status # 2) taskname + status + failure # 3) taskname + status + resubmitted + jobsetid # 4) taskname + status == (1) # 5) status + limit + getstatus + workername # 6) taskname + runs + lumis elif method in ['GET']: validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_num("limit", param, safe, optional=True) validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True) validate_num("subjobdef", param, safe, optional=True) validate_str("subuser", param, safe, RX_DN, optional=True) # possible combinations to check # 1) workername + getstatus + limit # 2) subresource + subjobdef + subuser elif method in ['DELETE']: pass
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: validate_str("workflow", param, safe, RX_WORKFLOW, optional=False) validate_num("subjobdef", param, safe, optional=True) validate_str("substatus", param, safe, RX_STATUS, optional=False) validate_strlist("subblocks", param, safe, RX_BLOCK) validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True) validate_str("subuser", param, safe, RX_DN, optional=False) elif method in ['POST']: validate_str("workflow", param, safe, RX_WORKFLOW, optional=True) validate_str("status", param, safe, RX_STATUS, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_num("jobset", param, safe, optional=True) validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True) validate_numlist("resubmittedjobs", param, safe) validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True) validate_num("limit", param, safe, optional=True) validate_strlist("runs", param, safe, RX_RUNS) validate_strlist("lumis", param, safe, RX_LUMIRANGE) # possible combinations to check # 1) taskname + status # 2) taskname + status + failure # 3) taskname + status + resubmitted + jobsetid # 4) taskname + status == (1) # 5) status + limit + getstatus + workername # 6) taskname + runs + lumis elif method in ['GET']: validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_num("limit", param, safe, optional=True) validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True) validate_num("subjobdef", param, safe, optional=True) validate_str("subuser", param, safe, RX_DN, optional=True) # possible combinations to check # 1) workername + getstatus + limit # 2) subresource + subjobdef + subuser elif method in ['DELETE']: pass
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: #TODO check optional parameter #TODO check all the regexp validate_str("taskname", param, safe, RX_WORKFLOW, optional=False) validate_strlist("outfilelumis", param, safe, RX_LUMILIST) validate_numlist("outfileruns", param, safe) if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]): raise InvalidParameter("The number of runs and the number of lumis lists are different") validate_strlist("inparentlfns", param, safe, RX_PARENTLFN) validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True) validate_num("pandajobid", param, safe, optional=False) validate_num("outsize", param, safe, optional=False) validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False) validate_str("appver", param, safe, RX_CMSSW, optional=False) validate_str("outtype", param, safe, RX_OUTTYPES, optional=False) validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False) validate_str("outlocation", param, safe, RX_CMSSITE, optional=False) validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False) validate_str("acquisitionera", param, safe, RX_WORKFLOW, optional=False)#TODO Do we really need this? validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex validate_str("outlfn", param, safe, RX_LFN, optional=False) validate_num("events", param, safe, optional=False) elif method in ['POST']: raise NotImplementedError elif method in ['GET']: validate_str("taskname", param, safe, RX_WORKFLOW, optional=False) validate_str("filetype", param, safe, RX_OUTTYPES, optional=False) elif method in ['DELETE']: raise NotImplementedError
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False) validate_strlist("workflow", param, safe, RX_TASKNAME) elif method in ['POST']: validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False) elif method in ['GET']: validate_str("campaign", param, safe, RX_CAMPAIGN, optional=True) validate_num('age', param, safe, optional=True) validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True) validate_num('limit', param, safe, optional=True) if not safe.kwargs['campaign'] and not safe.kwargs['age']: raise InvalidParameter("Invalid input parameters") if not safe.kwargs['campaign'] and safe.kwargs['subresource']: raise InvalidParameter("Invalid input parameters") elif method in ['DELETE']: validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False) validate_num("force", param, safe, optional=True)
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() # TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here. # Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource) if method in ["POST"]: validate_str("workflow", param, safe, RX_TASKNAME, optional=True) validate_str("status", param, safe, RX_STATUS, optional=True) validate_str("command", param, safe, RX_STATUS, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True) validate_numlist("resubmittedjobs", param, safe) validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True) validate_num("limit", param, safe, optional=True) validate_num("clusterid", param, safe, optional=True) # clusterid of the dag # possible combinations to check # 1) taskname + status # 2) taskname + status + failure # 3) taskname + status + resubmitted + jobsetid # 4) taskname + status == (1) # 5) status + limit + getstatus + workername # 6) taskname + runs + lumis elif method in ["GET"]: validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_num("limit", param, safe, optional=True) validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: #TODO check optional parameter #TODO check all the regexp validate_str("taskname", param, safe, RX_WORKFLOW, optional=False) validate_strlist("outfilelumis", param, safe, RX_LUMILIST) validate_numlist("outfileruns", param, safe) if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]): raise InvalidParameter("The number of runs and the number of lumis lists are different") validate_strlist("inparentlfns", param, safe, RX_PARENTLFN) validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True) validate_num("pandajobid", param, safe, optional=False) validate_num("outsize", param, safe, optional=False) validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False) validate_str("appver", param, safe, RX_CMSSW, optional=False) validate_str("outtype", param, safe, RX_OUTTYPES, optional=False) validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False) validate_str("outlocation", param, safe, RX_CMSSITE, optional=False) validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False) validate_str("acquisitionera", param, safe, RX_WORKFLOW, optional=False)#TODO Do we really need this? validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False) validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True) validate_num("events", param, safe, optional=False) validate_str("filestate", param, safe, RX_FILESTATE, optional=True) validate_num("directstageout", param, safe, optional=True) safe.kwargs["directstageout"] = 'T' if safe.kwargs["directstageout"] else 'F' #'F' if not provided elif method in ['POST']: validate_str("taskname", param, safe, RX_WORKFLOW, optional=False) validate_str("outlfn", param, safe, RX_LFN, optional=False) validate_str("filestate", param, safe, RX_FILESTATE, optional=False) elif method in ['GET']: validate_str("taskname", param, safe, RX_WORKFLOW, optional=False) validate_str("filetype", param, safe, RX_OUTTYPES, optional=False) elif method in ['DELETE']: authz_operator() validate_str("taskname", param, safe, RX_WORKFLOW, optional=True) validate_str("hours", param, safe, RX_HOURS, optional=True) if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]): raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\ " will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if not (authz_operator_without_raise('aso', 'operator') or authz_operator_without_raise('crab3', 'operator')): # TODO: do not forget to uncomment this # so far just print a warning print ('WARNING. NOT AUTHORIZED') #raise ForbiddenAccess('Access is restricted to this API') if method in ['PUT']: raise UnsupportedMethod('This method is not supported in this API!') elif method in ['POST']: # POST is for update, so we should allow anyone anything? # Of Course no, but there are multiple combinations, so we are not validating here # and all validation is in post function validate_str("subresource", param, safe, RX_SUBPOSTTRANSFER, optional=False) validate_str("asoworker", param, safe, RX_ASO_WORKERNAME, optional=False) validate_str("username", param, safe, RX_USERNAME, optional=True) validate_str("list_of_ids", param, safe, RX_ANYTHING, optional=True) validate_str("list_of_transfer_state", param, safe, RX_ANYTHING, optional=True) validate_str("list_of_failure_reason", param, safe, RX_ANYTHING, optional=True) validate_str("list_of_retry_value", param, safe, RX_ANYTHING, optional=True) validate_str("list_of_fts_instance", param, safe, RX_ANYTHING, optional=True) validate_str("list_of_fts_id", param, safe, RX_ANYTHING, optional=True) validate_str("list_of_publication_state", param, safe, RX_ANYTHING, optional=True) validate_num("time_to", param, safe, optional=True) validate_num("publish_flag", param, safe, optional=True) elif method in ['GET']: validate_str("subresource", param, safe, RX_SUBGETTRANSFER, optional=False) validate_str("username", param, safe, RX_USERNAME, optional=True) validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True) validate_str("vorole", param, safe, RX_VOPARAMS, optional=True) validate_str("taskname", param, safe, RX_TASKNAME, optional=True) validate_str("destination", param, safe, RX_CMSSITE, optional=True) validate_str("source", param, safe, RX_CMSSITE, optional=True) validate_str("asoworker", param, safe, RX_ASO_WORKERNAME, optional=True) validate_num("grouping", param, safe, optional=False) validate_num("limit", param, safe, optional=True) elif method in ['DELETE']: # This one I don`t really like to have implemented # There is some security concerns # But I still have one idea what we should clean up and where. # So leave it for the future. pass
def validate(apiobj, method, api, param, safe): #pylint: disable=unused-argument """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid( ) #TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here. # Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource) if method in ['POST']: validate_str("workflow", param, safe, RX_TASKNAME, optional=True) validate_str("status", param, safe, RX_STATUS, optional=True) validate_str("command", param, safe, RX_STATUS, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True) validate_strlist("resubmittedjobs", param, safe, RX_JOBID) validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True) validate_num("limit", param, safe, optional=True) validate_num("clusterid", param, safe, optional=True) #clusterid of the dag # possible combinations to check # 1) taskname + status # 2) taskname + status + failure # 3) taskname + status + resubmitted + jobsetid # 4) taskname + status == (1) # 5) status + limit + getstatus + workername # 6) taskname + runs + lumis elif method in ['GET']: validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_num("limit", param, safe, optional=True) validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False) validate_strlist("workflow", param, safe, RX_WORKFLOW) elif method in ['POST']: validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False) elif method in ['GET']: validate_str("campaign", param, safe, RX_CAMPAIGN, optional=True) validate_num('age', param, safe, optional=True) validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True) validate_num('limit', param, safe, optional=True) if not safe.kwargs['campaign'] and not safe.kwargs['age']: raise InvalidParameter("Invalid input parameters") if not safe.kwargs['campaign'] and safe.kwargs['subresource']: raise InvalidParameter("Invalid input parameters") elif method in ['DELETE']: validate_str("campaign", param, safe, RX_CAMPAIGN, optional=False) validate_num("force", param, safe, optional=True)
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" #authz_login_valid() if method in ['PUT']: validate_str("activity", param, safe, RX_ACTIVITY, optional=True) validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False) # TODO this should be changed to be non-optional validate_str("generator", param, safe, RX_GENERATOR, optional=True) validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True) validate_str("jobsw", param, safe, RX_CMSSW, optional=False) validate_num("nonprodsw", param, safe, optional=False) validate_str("jobarch", param, safe, RX_ARCH, optional=False) if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw']) jobtype = safe.kwargs.get('jobtype', None) validate_num("useparent", param, safe, optional=True) validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist']) validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist']) validate_str("splitalgo", param, safe, RX_SPLIT, optional=False) validate_num("algoargs", param, safe, optional=False) validate_num("totalunits", param, safe, optional=True) validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False) validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False) validate_str("lfn", param, safe, RX_LFN, optional=True) self._checkOutLFN(safe.kwargs) validate_strlist("addoutputfiles", param, safe, RX_ADDFILE) validate_strlist("userfiles", param, safe, RX_USERFILE) validate_num("savelogsflag", param, safe, optional=False) validate_num("saveoutput", param, safe, optional=True) validate_num("faillimit", param, safe, optional=True) validate_num("ignorelocality", param, safe, optional=True) if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []): safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist']) validate_str("vorole", param, safe, RX_VOPARAMS, optional=True) validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True) validate_num("publication", param, safe, optional=False) validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication']))) ## Validations needed in case publication is on. if safe.kwargs['publication']: ## The (user specified part of the) publication dataset name must be ## specified and must pass DBS validation. Since this is the correct ## validation function, it must be done before the ## validate_str("workflow", ...) and validate_str("publishname", ...) ## we have below. Not sure why RX_PUBLISH was introduced in CRAB, but ## it is not the correct regular expression for publication dataset ## name validation. self._checkPublishDataName(param.kwargs, safe.kwargs['lfn']) ## 'publishname' was already validated above in _checkPublishDataName(). ## But I am not sure if we can skip the validate_str('publishname', ...) ## or we need it so that all parameters are moved from param to safe. ## Therefore, I didn't remove the validate_str('publishname', ...), ## but only changed RX_PUBLISH -> RX_ANYTHING. validate_str('publishname', param, safe, RX_ANYTHING, optional=False) ## If this is a MC generation job type, the primary dataset name will ## be used to define the primary dataset field in the publication ## dataset name. Therefore, the primary dataset name must pass DBS ## validation. Since in a MC generation job type the primary dataset ## name is saved in 'inputdata', _checkPrimaryDataset() actually ## validates 'inputdata'. Therefore, this function must come before ## the validate_str('inputdata', ...) we have below. OTOH, matching ## 'inputdata' agains RX_DATASET is not the correct validation for ## the primary dataset name. if jobtype == 'PrivateMC': self._checkPrimaryDataset(param.kwargs) else: ## Not sure if we need to match publishname against RX_PUBLISH... validate_str("publishname", param, safe, RX_PUBLISH, optional=False) validate_num("publishgroupname", param, safe, optional=True) ## This line must come after _checkPublishDataName() validate_str("workflow", param, safe, RX_WORKFLOW, optional=False) if jobtype == 'Analysis': validate_str("inputdata", param, safe, RX_DATASET, optional=False) ## Added this if, because in this case 'inputdata' was already validated ## above in _checkPrimaryDataset(), and I am not sure if we can skip the ## validate_str('inputdata', ...) or we need it so that all parameters ## are moved from param to safe. elif jobtype == 'PrivateMC' and safe.kwargs['publication']: validate_str("inputdata", param, safe, RX_ANYTHING, optional=False) else: validate_str("inputdata", param, safe, RX_DATASET, optional=True) validate_num("nonvaliddata", param, safe, optional=True) #if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them) validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False) self._checkASODestination(safe.kwargs['asyncdest']) # We no longer use this attribute, but keep it around for older client compatibility validate_num("blacklistT1", param, safe, optional=True) validate_num("oneEventMode", param, safe, optional=True) validate_num("priority", param, safe, optional=True) validate_num("maxjobruntime", param, safe, optional=True) validate_num("numcores", param, safe, optional=True) validate_num("maxmemory", param, safe, optional=True) validate_str("dbsurl", param, safe, RX_DBSURL, optional=False) validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES) validate_strlist("edmoutfiles", param, safe, RX_OUTFILES) validate_strlist("runs", param, safe, RX_RUNS) validate_strlist("lumis", param, safe, RX_LUMIRANGE) #validate_str("scheduler", param, safe, RX_SCHEDULER) if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]): raise InvalidParameter("The number of runs and the number of lumis lists are different") validate_strlist("adduserfiles", param, safe, RX_ADDFILE) validate_str("asourl", param, safe, RX_ASOURL, optional=True) validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True) validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS) validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True) validate_str("collector", param, safe, RX_COLLECTOR, optional=True) validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS) validate_num("dryrun", param, safe, optional=True) elif method in ['POST']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False) validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True) ## In a resubmission, the site black- and whitelists need to be interpreted ## differently than in an initial task submission. If there is no site black- ## or whitelist, set it to None and DataWorkflow will use the corresponding ## list defined in the initial task submission. If the site black- or whitelist ## is equal to the string 'empty', set it to an empty list and don't call ## validate_strlist as it would fail. if 'siteblacklist' not in param.kwargs: safe.kwargs['siteblacklist'] = None elif param.kwargs['siteblacklist'] == 'empty': safe.kwargs['siteblacklist'] = [] del param.kwargs['siteblacklist'] else: validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist']) if 'sitewhitelist' not in param.kwargs: safe.kwargs['sitewhitelist'] = None elif param.kwargs['sitewhitelist'] == 'empty': safe.kwargs['sitewhitelist'] = [] del param.kwargs['sitewhitelist'] else: validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist']) validate_numlist('jobids', param, safe) validate_num("priority", param, safe, optional=True) validate_num("maxjobruntime", param, safe, optional=True) validate_num("numcores", param, safe, optional=True) validate_num("maxmemory", param, safe, optional=True) validate_num("force", param, safe, optional=True) elif method in ['GET']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True) validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True) validate_str('username', param, safe, RX_USERNAME, optional=True) validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric ## Used to determine how much information to return to the client for status. ## also used by report to determine if it has to check job states validate_num("verbose", param, safe, optional=True) ## used by get log, get data validate_num('limit', param, safe, optional=True) validate_num('exitcode', param, safe, optional=True) validate_numlist('jobids', param, safe) ## used by errors and report (short format in report means we do not query DBS) validate_num('shortformat', param, safe, optional=True) ## validation parameters if not safe.kwargs['workflow'] and safe.kwargs['subresource']: raise InvalidParameter("Invalid input parameters") if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']: raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.") elif method in ['DELETE']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False) validate_num("force", param, safe, optional=True) validate_numlist('jobids', param, safe)
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid( ) #TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here. # Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource) if method in ['PUT']: validate_str("workflow", param, safe, RX_TASKNAME, optional=False) validate_num("subjobdef", param, safe, optional=True) validate_str("substatus", param, safe, RX_STATUS, optional=False) validate_strlist("subblocks", param, safe, RX_BLOCK) validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True) validate_str("subuser", param, safe, RX_DN, optional=False) elif method in ['POST']: validate_str("workflow", param, safe, RX_TASKNAME, optional=True) validate_str("status", param, safe, RX_STATUS, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_num("jobset", param, safe, optional=True) validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True) validate_numlist("resubmittedjobs", param, safe) validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True) validate_num("limit", param, safe, optional=True) validate_strlist("runs", param, safe, RX_RUNS) validate_strlist("lumis", param, safe, RX_LUMIRANGE) # possible combinations to check # 1) taskname + status # 2) taskname + status + failure # 3) taskname + status + resubmitted + jobsetid # 4) taskname + status == (1) # 5) status + limit + getstatus + workername # 6) taskname + runs + lumis elif method in ['GET']: validate_str("workername", param, safe, RX_WORKER_NAME, optional=True) validate_str("getstatus", param, safe, RX_STATUS, optional=True) validate_num("limit", param, safe, optional=True) validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True) validate_num("subjobdef", param, safe, optional=True) validate_str("subuser", param, safe, RX_DN, optional=True) # possible combinations to check # 1) workername + getstatus + limit # 2) subresource + subjobdef + subuser elif method in ['DELETE']: pass
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: #TODO check optional parameter #TODO check all the regexp validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_strlist("outfilelumis", param, safe, RX_LUMILIST) validate_numlist("outfileruns", param, safe) if len(safe.kwargs["outfileruns"]) != len( safe.kwargs["outfilelumis"]): raise InvalidParameter( "The number of runs and the number of lumis lists are different" ) validate_strlist("inparentlfns", param, safe, RX_PARENTLFN) validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True) validate_str("jobid", param, safe, RX_JOBID, optional=True) #TODO: for backward compatibility. Get rid of the pandajobid once all jobs using it are done (one month after this gets released) #(The following five lines can be deleted) validate_num("pandajobid", param, safe, optional=True) if bool(safe.kwargs["jobid"]) == bool(safe.kwargs["pandajobid"]): raise InvalidParameter( "Only one among jobid and pandajobid should be set") #Oracle/cx_oracle/python stack does not like None for numbers, even if they are nullable if safe.kwargs["pandajobid"] == None: safe.kwargs["pandajobid"] = 0 validate_num("outsize", param, safe, optional=False) validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False) validate_str("appver", param, safe, RX_CMSSW, optional=False) validate_str("outtype", param, safe, RX_OUTTYPES, optional=False) validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False) validate_str("outlocation", param, safe, RX_CMSSITE, optional=False) validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False) validate_str("acquisitionera", param, safe, RX_TASKNAME, optional=False) #TODO Do we really need this? validate_str( "outdatasetname", param, safe, RX_OUTDSLFN, optional=False) #TODO temporary, need to come up with a regex validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False) validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True) validate_num("events", param, safe, optional=False) validate_str("filestate", param, safe, RX_FILESTATE, optional=True) validate_num("directstageout", param, safe, optional=True) safe.kwargs["directstageout"] = 'T' if safe.kwargs[ "directstageout"] else 'F' #'F' if not provided elif method in ['POST']: validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_str("outlfn", param, safe, RX_LFN, optional=False) validate_str("filestate", param, safe, RX_FILESTATE, optional=False) elif method in ['GET']: validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_str("filetype", param, safe, RX_OUTTYPES, optional=False) validate_num("howmany", param, safe, optional=True) validate_strlist("lfn", param, safe, RX_LFN) elif method in ['DELETE']: authz_operator() validate_str("taskname", param, safe, RX_TASKNAME, optional=True) validate_str("hours", param, safe, RX_HOURS, optional=True) if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]): raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\ " will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" #authz_login_valid() if method in ['PUT']: validate_str("workflow", param, safe, RX_WORKFLOW, optional=False) validate_str("activity", param, safe, RX_ACTIVITY, optional=True) validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False) validate_str("generator", param, safe, RX_GENERATOR, optional=True) validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True) validate_str("jobsw", param, safe, RX_CMSSW, optional=False) validate_num("nonprodsw", param, safe, optional=False) validate_str("jobarch", param, safe, RX_ARCH, optional=False) if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw']) jobtype = safe.kwargs.get('jobtype', None) if jobtype == 'Analysis': validate_str("inputdata", param, safe, RX_DATASET, optional=False) else: validate_str("inputdata", param, safe, RX_DATASET, optional=True) validate_num("useparent", param, safe, optional=True) validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist']) validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist']) validate_str("splitalgo", param, safe, RX_SPLIT, optional=False) validate_num("algoargs", param, safe, optional=False) validate_num("totalunits", param, safe, optional=True) validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False) validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False) validate_str("lfnprefix", param, safe, RX_LFNPATH, optional=True) #keeping this for compatibility with old clients. Will be converted into a full lfn validate_str("lfn", param, safe, RX_LFN, optional=True) self._checkOutLFN(safe.kwargs) validate_strlist("addoutputfiles", param, safe, RX_ADDFILE) validate_strlist("userfiles", param, safe, RX_USERFILE) validate_num("savelogsflag", param, safe, optional=False) validate_num("saveoutput", param, safe, optional=True) validate_num("faillimit", param, safe, optional=True) validate_num("ignorelocality", param, safe, optional=True) if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []): safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist']) validate_str("vorole", param, safe, RX_VOPARAMS, optional=True) validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True) validate_str("publishname", param, safe, RX_PUBLISH, optional=False) validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=True) validate_num("publication", param, safe, optional=False) #if publication is set as true both publishDataName and publishDbsUrl are needed if safe.kwargs["publication"] and not (bool(safe.kwargs["publishname"]) and bool(safe.kwargs["publishdbsurl"])): raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication") #if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them) validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False) self._checkASODestination(safe.kwargs['asyncdest']) # We no longer use this attribute, but keep it around for older client compatibility validate_num("blacklistT1", param, safe, optional=True) validate_num("oneEventMode", param, safe, optional=True) validate_num("priority", param, safe, optional=True) validate_num("maxjobruntime", param, safe, optional=True) validate_num("numcores", param, safe, optional=True) validate_num("maxmemory", param, safe, optional=True) validate_str("dbsurl", param, safe, RX_DBSURL, optional=False) validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES) validate_strlist("edmoutfiles", param, safe, RX_OUTFILES) validate_strlist("runs", param, safe, RX_RUNS) validate_strlist("lumis", param, safe, RX_LUMIRANGE) #validate_str("scheduler", param, safe, RX_SCHEDULER) if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]): raise InvalidParameter("The number of runs and the number of lumis lists are different") validate_strlist("adduserfiles", param, safe, RX_ADDFILE) validate_str("asourl", param, safe, RX_ASOURL, optional=True) validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True) validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS) validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True) validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS) elif method in ['POST']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False) validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist']) validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist']) validate_numlist('jobids', param, safe) validate_num("priority", param, safe, optional=True) validate_num("maxjobruntime", param, safe, optional=True) validate_num("numcores", param, safe, optional=True) validate_num("maxmemory", param, safe, optional=True) elif method in ['GET']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True) validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True) validate_str('username', param, safe, RX_USERNAME, optional=True) validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric ## Used to determine how much information to return to the client for status. ## also used by report to determine if it has to check job states validate_num("verbose", param, safe, optional=True) ## used by get log, get data validate_num('limit', param, safe, optional=True) validate_num('exitcode', param, safe, optional=True) validate_numlist('jobids', param, safe) ## used by errors and report (short format in report means we do not query DBS) validate_num('shortformat', param, safe, optional=True) ## validation parameters if not safe.kwargs['workflow'] and safe.kwargs['subresource']: raise InvalidParameter("Invalid input parameters") if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']: raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.") elif method in ['DELETE']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False) validate_num("force", param, safe, optional=True) validate_numlist('jobids', param, safe)
def validate(self, apiobj, method, api, param, safe): #pylint: disable=unused-argument """Validating all the input parameter as enforced by the WMCore.REST module""" if method in ['PUT']: username = cherrypy.request.user['login'] # username registered in CMS WEB frontend requestname = param.kwargs['workflow'] param.kwargs['workflow'] = generateTaskName(username, requestname) validate_str("workflow", param, safe, RX_TASKNAME, optional=False) validate_str("activity", param, safe, RX_ACTIVITY, optional=True) validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False) # TODO this should be changed to be non-optional validate_str("generator", param, safe, RX_GENERATOR, optional=True) validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True) validate_str("jobsw", param, safe, RX_CMSSW, optional=False) validate_num("nonprodsw", param, safe, optional=False) validate_str("jobarch", param, safe, RX_ARCH, optional=False) if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw']) validate_num("useparent", param, safe, optional=True) validate_str("secondarydata", param, safe, RX_DATASET, optional=True) validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist']) validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist']) validate_str("splitalgo", param, safe, RX_SPLIT, optional=False) validate_num("algoargs", param, safe, optional=False) try: validate_num("totalunits", param, safe, optional=True) except InvalidParameter: validate_real("totalunits", param, safe, optional=True) validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False) validate_str("debugfilename", param, safe, RX_CACHENAME, optional=True) validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False) validate_str("lfn", param, safe, RX_LFN, optional=True) self._checkOutLFN(safe.kwargs, username) validate_strlist("addoutputfiles", param, safe, RX_ADDFILE, custom_err="Incorrect 'JobType.outputFiles' parameter. " \ "Allowed regexp for each filename: '%s'." % RX_ADDFILE.pattern) validate_strlist("userfiles", param, safe, RX_USERFILE, custom_err="Incorrect 'Data.userInputFiles' parameter. " \ "Allowed regexp for each filename: '%s'." % RX_USERFILE.pattern) validate_num("savelogsflag", param, safe, optional=False) validate_num("saveoutput", param, safe, optional=True) validate_num("faillimit", param, safe, optional=True) validate_num("ignorelocality", param, safe, optional=True) if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []): safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist']) validate_str("vorole", param, safe, RX_VOPARAMS, optional=True) validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True) validate_num("publication", param, safe, optional=False) validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication']))) ## We might want to remove publishname once the backward compatibility ## wont be needed anymore. Then we can just keep publishname2 ## NB: AFAIK the only client not using the CRABLibrary but direct cutl is HC, ## therefore we will need to make sure we do not break it! ## The following two lines will be removed in the future once we will ## not need backward compatibility anymore self._checkPublishDataName(param.kwargs, safe.kwargs['lfn'], requestname, username) validate_str('publishname', param, safe, RX_ANYTHING, optional=True) ##And this if as well, just do self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username) if not safe.kwargs["publishname"]: #new clients won't define this anymore ## The (user specified part of the) publication dataset name must be ## specified and must pass DBS validation. Since this is the correct ## validation function, it must be done before the ## validate_str("publishname", ...) we have below. self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username) else: param.kwargs["publishname2"] = safe.kwargs["publishname"] ## 'publishname' was already validated above in _checkPublishDataName(). ## Calling validate_str with a fake regexp to move the param to the ## list of validated inputs validate_str("publishname2", param, safe, RX_ANYTHING, optional=True) validate_num("publishgroupname", param, safe, optional=True) if safe.kwargs['jobtype'] == 'PrivateMC': if param.kwargs['inputdata']: msg = "Invalid 'inputdata' parameter." msg += " Job type PrivateMC does not take any input dataset." msg += " If you really intend to run over an input dataset, then you must use job type Analysis." raise InvalidParameter(msg) if safe.kwargs['userfiles']: msg = "Invalid 'userfiles' parameter." msg += " Job type PrivateMC does not take any input files." msg += " If you really intend to run over input files, then you must use job type Analysis." raise InvalidParameter(msg) ## Client versions < 3.3.1511 may put in the input dataset something that is not ## really an input dataset (for PrivateMC or user input files). So the only case ## in which we are sure that we have to validate the input dataset is when the ## workflow type is Analysis, the workflow does not run on user input files and ## an input dataset is defined (scriptExe may not define an input). ## Once we don't care anymore about backward compatibility with client < 3.3.1511, ## we can uncomment the 1st line below and delete the next 4 lines. #validate_str("inputdata", param, safe, RX_DATASET, optional=True) if safe.kwargs['jobtype'] == 'Analysis' and not safe.kwargs['userfiles'] and 'inputdata' in param.kwargs: validate_str("inputdata", param, safe, RX_DATASET, optional=True) else: validate_str("inputdata", param, safe, RX_ANYTHING, optional=True) ## The client is not forced to define the primary dataset. So make sure to have ## defaults or take it from the input dataset. The primary dataset is needed for ## the LFN of the output/log files and for publication. We want to have it well ## defined even if publication and/or transfer to storage are off. if safe.kwargs['inputdata']: param.kwargs['primarydataset'] = safe.kwargs['inputdata'].split('/')[1] if not param.kwargs.get('primarydataset', None): if safe.kwargs['jobtype'] == 'PrivateMC': param.kwargs['primarydataset'] = "CRAB_PrivateMC" elif safe.kwargs['jobtype'] == 'Analysis' and safe.kwargs['userfiles']: param.kwargs['primarydataset'] = "CRAB_UserFiles" else: param.kwargs['primarydataset'] = "CRAB_NoInput" ## We validate the primary dataset agains DBS rules even if publication is off, ## because in the future we may want to give the possibility to users to publish ## a posteriori. self._checkPrimaryDataset(param.kwargs, optional=False) validate_str("primarydataset", param, safe, RX_LFNPRIMDS, optional=False) validate_num("nonvaliddata", param, safe, optional=True) #if one and only one between outputDatasetTag and publishDbsUrl is set raise an error (we need both or none of them) validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False) self._checkASODestination(safe.kwargs['asyncdest']) # We no longer use this attribute, but keep it around for older client compatibility validate_num("blacklistT1", param, safe, optional=True) validate_num("oneEventMode", param, safe, optional=True) validate_num("priority", param, safe, optional=True) validate_num("maxjobruntime", param, safe, optional=True) validate_num("numcores", param, safe, optional=True) validate_num("maxmemory", param, safe, optional=True) validate_str("dbsurl", param, safe, RX_DBSURL, optional=False) validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect tfileoutfiles parameter (TFileService). " \ "Allowed regexp: '%s'." % RX_OUTFILES.pattern) validate_strlist("edmoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect edmoutfiles parameter (PoolOutputModule). " \ "Allowed regexp: '%s'." % RX_OUTFILES.pattern) validate_strlist("runs", param, safe, RX_RUNS) validate_strlist("lumis", param, safe, RX_LUMIRANGE) if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]): raise InvalidParameter("The number of runs and the number of lumis lists are different") validate_strlist("adduserfiles", param, safe, RX_ADDFILE) validate_str("asourl", param, safe, RX_ASOURL, optional=True) validate_str("asodb", param, safe, RX_ASODB, optional=True) safe.kwargs["asourl"], safe.kwargs["asodb"] = self._getAsoConfig(safe.kwargs["asourl"], safe.kwargs["asodb"]) validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True) validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS) validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True) validate_str("collector", param, safe, RX_COLLECTOR, optional=True) validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS) validate_num("dryrun", param, safe, optional=True) validate_num("ignoreglobalblacklist", param, safe, optional=True) elif method in ['POST']: validate_str("workflow", param, safe, RX_TASKNAME, optional=False) validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True) validate_strlist('jobids', param, safe, RX_JOBID) ## In a resubmission, the site black- and whitelists need to be interpreted ## differently than in an initial task submission. If there is no site black- ## or whitelist, set it to None and DataWorkflow will use the corresponding ## list defined in the initial task submission. If the site black- or whitelist ## is equal to the string 'empty', set it to an empty list and don't call ## validate_strlist as it would fail. if 'siteblacklist' not in param.kwargs: safe.kwargs['siteblacklist'] = None elif param.kwargs['siteblacklist'] == 'empty': safe.kwargs['siteblacklist'] = [] del param.kwargs['siteblacklist'] else: validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist']) if 'sitewhitelist' not in param.kwargs: safe.kwargs['sitewhitelist'] = None elif param.kwargs['sitewhitelist'] == 'empty': safe.kwargs['sitewhitelist'] = [] del param.kwargs['sitewhitelist'] else: validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist']) validate_num("maxjobruntime", param, safe, optional=True) validate_num("maxmemory", param, safe, optional=True) validate_num("numcores", param, safe, optional=True) validate_num("priority", param, safe, optional=True) validate_num("force", param, safe, optional=True) validate_num("publication", param, safe, optional=True) elif method in ['GET']: validate_str("workflow", param, safe, RX_TASKNAME, optional=True) validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True) validate_str('username', param, safe, RX_USERNAME, optional=True) validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric ## Used to determine how much information to return to the client for status. ## also used by report to determine if it has to check job states validate_num("verbose", param, safe, optional=True) ## used by get log, get data validate_num('limit', param, safe, optional=True) validate_num('exitcode', param, safe, optional=True) validate_strlist('jobids', param, safe, RX_JOBID) ## used by errors and report (short format in report means we do not query DBS) validate_num('shortformat', param, safe, optional=True) # used by publicationStatus validate_str("asourl", param, safe, RX_ASOURL, optional=True) validate_str("asodb", param, safe, RX_ASODB, optional=True) ## validation parameters if not safe.kwargs['workflow'] and safe.kwargs['subresource']: raise InvalidParameter("Invalid input parameters") if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']: raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.") elif method in ['DELETE']: validate_str("workflow", param, safe, RX_TASKNAME, optional=False) validate_num("force", param, safe, optional=True) validate_str("killwarning", param, safe, RX_TEXT_FAIL, optional=True) #decode killwarning message if present if safe.kwargs['killwarning']: try: safe.kwargs['killwarning'] = b64decode(safe.kwargs['killwarning']) except TypeError: raise InvalidParameter("Failure message is not in the accepted format")
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: #TODO check optional parameter #TODO check all the regexp validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_strlist("outfilelumis", param, safe, RX_LUMILIST) validate_numlist("outfileruns", param, safe) if len(safe.kwargs["outfileruns"]) != len( safe.kwargs["outfilelumis"]): raise InvalidParameter( "The number of runs and the number of lumis lists are different" ) validate_strlist("inparentlfns", param, safe, RX_PARENTLFN) validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True) validate_num("pandajobid", param, safe, optional=False) validate_num("outsize", param, safe, optional=False) validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False) validate_str("appver", param, safe, RX_CMSSW, optional=False) validate_str("outtype", param, safe, RX_OUTTYPES, optional=False) validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False) validate_str("outlocation", param, safe, RX_CMSSITE, optional=False) validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False) validate_str("acquisitionera", param, safe, RX_TASKNAME, optional=False) #TODO Do we really need this? validate_str( "outdatasetname", param, safe, RX_OUTDSLFN, optional=False) #TODO temporary, need to come up with a regex validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False) validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True) validate_num("events", param, safe, optional=False) validate_str("filestate", param, safe, RX_FILESTATE, optional=True) validate_num("directstageout", param, safe, optional=True) safe.kwargs["directstageout"] = 'T' if safe.kwargs[ "directstageout"] else 'F' #'F' if not provided elif method in ['POST']: validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_str("outlfn", param, safe, RX_LFN, optional=False) validate_str("filestate", param, safe, RX_FILESTATE, optional=False) elif method in ['GET']: validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_str("filetype", param, safe, RX_OUTTYPES, optional=False) elif method in ['DELETE']: authz_operator() validate_str("taskname", param, safe, RX_TASKNAME, optional=True) validate_str("hours", param, safe, RX_HOURS, optional=True) if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]): raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\ " will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: #TODO check optional parameter #TODO check all the regexp validate_str("taskname", param, safe, RX_WORKFLOW, optional=False) validate_strlist("outfilelumis", param, safe, RX_LUMILIST) validate_numlist("outfileruns", param, safe) if len(safe.kwargs["outfileruns"]) != len( safe.kwargs["outfilelumis"]): raise InvalidParameter( "The number of runs and the number of lumis lists are different" ) validate_strlist("inparentlfns", param, safe, RX_PARENTLFN) validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True) validate_num("pandajobid", param, safe, optional=False) validate_num("outsize", param, safe, optional=False) validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False) validate_str("appver", param, safe, RX_CMSSW, optional=False) validate_str("outtype", param, safe, RX_OUTTYPES, optional=False) validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False) validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False) validate_str("outlocation", param, safe, RX_CMSSITE, optional=False) validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False) validate_str("acquisitionera", param, safe, RX_WORKFLOW, optional=False) #TODO Do we really need this? validate_str( "outdatasetname", param, safe, RX_OUTDSLFN, optional=False) #TODO temporary, need to come up with a regex validate_str("outlfn", param, safe, RX_LFN, optional=False) validate_num("events", param, safe, optional=False) elif method in ['POST']: raise NotImplementedError elif method in ['GET']: validate_str("taskname", param, safe, RX_WORKFLOW, optional=False) validate_str("filetype", param, safe, RX_OUTTYPES, optional=False) elif method in ['DELETE']: raise NotImplementedError
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_strlist("outfilelumis", param, safe, RX_LUMILIST) validate_strlist("outfileruns", param, safe, RX_RUNS) if len(safe.kwargs["outfileruns"]) != len( safe.kwargs["outfilelumis"]): raise InvalidParameter( "The number of runs and the number of lumis lists are different" ) validate_strlist("inparentlfns", param, safe, RX_PARENTLFN) # inparentlfns will be inserted in Oracle as CLOB, so it must be a string safe.kwargs['inparentlfns'] = str(safe.kwargs['inparentlfns']) validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True) validate_str("jobid", param, safe, RX_JOBID, optional=True) safe.kwargs["pandajobid"] = 0 validate_num("outsize", param, safe, optional=False) validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False) validate_str("appver", param, safe, RX_CMSSW, optional=False) validate_str("outtype", param, safe, RX_OUTTYPES, optional=False) validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False) validate_num("checksumcksum", param, safe, optional=False) validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False) validate_str("outlocation", param, safe, RX_CMSSITE, optional=False) validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False) validate_str("acquisitionera", param, safe, RX_TASKNAME, optional=False) validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False) # need to use RX_PARENTLFN becasue same API is also used for input metadata validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False) validate_str("outtmplfn", param, safe, RX_LFN, optional=True) validate_num("events", param, safe, optional=False) validate_str("filestate", param, safe, RX_FILESTATE, optional=True) validate_num("directstageout", param, safe, optional=True) safe.kwargs["directstageout"] = 'T' if safe.kwargs[ "directstageout"] else 'F' #'F' if not provided elif method in ['POST']: validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_str("outlfn", param, safe, RX_LFN, optional=False) validate_str("filestate", param, safe, RX_FILESTATE, optional=False) elif method in ['GET']: validate_str("taskname", param, safe, RX_TASKNAME, optional=False) validate_str("filetype", param, safe, RX_OUTTYPES, optional=False) validate_num("howmany", param, safe, optional=True) validate_strlist("lfn", param, safe, RX_LFN) elif method in ['DELETE']: authz_operator() validate_str("taskname", param, safe, RX_TASKNAME, optional=True) validate_str("hours", param, safe, RX_HOURS, optional=True) if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]): raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\ " will be deleted. Only one of the two parameters can be specified.")
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" #authz_login_valid() if method in ['PUT']: validate_str("workflow", param, safe, RX_WORKFLOW, optional=False) validate_str("activity", param, safe, RX_ACTIVITY, optional=True) validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False) validate_str("generator", param, safe, RX_GENERATOR, optional=True) validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True) validate_str("jobsw", param, safe, RX_CMSSW, optional=False) validate_num("nonprodsw", param, safe, optional=False) validate_str("jobarch", param, safe, RX_ARCH, optional=False) if not safe.kwargs[ "nonprodsw"]: #if the user wants to allow non-production releases self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw']) jobtype = safe.kwargs.get('jobtype', None) if jobtype == 'Analysis': validate_str("inputdata", param, safe, RX_DATASET, optional=False) else: validate_str("inputdata", param, safe, RX_DATASET, optional=True) validate_num("useparent", param, safe, optional=True) validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites( safe.kwargs['siteblacklist']) validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites( safe.kwargs['sitewhitelist']) validate_str("splitalgo", param, safe, RX_SPLIT, optional=False) validate_num("algoargs", param, safe, optional=False) validate_num("totalunits", param, safe, optional=True) validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False) validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False) validate_str( "lfnprefix", param, safe, RX_LFNPATH, optional=True ) #keeping this for compatibility with old clients. Will be converted into a full lfn validate_str("lfn", param, safe, RX_LFN, optional=True) self._checkOutLFN(safe.kwargs) validate_strlist("addoutputfiles", param, safe, RX_ADDFILE) validate_strlist("userfiles", param, safe, RX_USERFILE) validate_num("savelogsflag", param, safe, optional=False) validate_num("saveoutput", param, safe, optional=True) validate_num("faillimit", param, safe, optional=True) validate_num("ignorelocality", param, safe, optional=True) if safe.kwargs[ 'ignorelocality'] and self.centralcfg.centralconfig.get( 'ign-locality-blacklist', []): safe.kwargs['siteblacklist'] += self._expandSites( self.centralcfg.centralconfig['ign-locality-blacklist']) validate_str("vorole", param, safe, RX_VOPARAMS, optional=True) validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True) validate_str("publishname", param, safe, RX_PUBLISH, optional=False) validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=True) validate_num("publication", param, safe, optional=False) #if publication is set as true both publishDataName and publishDbsUrl are needed if safe.kwargs["publication"] and not ( bool(safe.kwargs["publishname"]) and bool(safe.kwargs["publishdbsurl"])): raise InvalidParameter( "You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication" ) #if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them) validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False) self._checkASODestination(safe.kwargs['asyncdest']) # We no longer use this attribute, but keep it around for older client compatibility validate_num("blacklistT1", param, safe, optional=True) validate_num("oneEventMode", param, safe, optional=True) validate_num("priority", param, safe, optional=True) validate_num("maxjobruntime", param, safe, optional=True) validate_num("numcores", param, safe, optional=True) validate_num("maxmemory", param, safe, optional=True) validate_str("dbsurl", param, safe, RX_DBSURL, optional=False) validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES) validate_strlist("edmoutfiles", param, safe, RX_OUTFILES) validate_strlist("runs", param, safe, RX_RUNS) validate_strlist("lumis", param, safe, RX_LUMIRANGE) #validate_str("scheduler", param, safe, RX_SCHEDULER) if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]): raise InvalidParameter( "The number of runs and the number of lumis lists are different" ) validate_strlist("adduserfiles", param, safe, RX_ADDFILE) validate_str("asourl", param, safe, RX_ASOURL, optional=True) validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True) validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS) validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True) validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS) elif method in ['POST']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False) validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites( safe.kwargs['siteblacklist']) validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites( safe.kwargs['sitewhitelist']) validate_numlist('jobids', param, safe) validate_num("priority", param, safe, optional=True) validate_num("maxjobruntime", param, safe, optional=True) validate_num("numcores", param, safe, optional=True) validate_num("maxmemory", param, safe, optional=True) elif method in ['GET']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True) validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True) validate_str('username', param, safe, RX_USERNAME, optional=True) validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric ## Used to determine how much information to return to the client for status. ## also used by report to determine if it has to check job states validate_num("verbose", param, safe, optional=True) ## used by get log, get data validate_num('limit', param, safe, optional=True) validate_num('exitcode', param, safe, optional=True) validate_numlist('jobids', param, safe) ## used by errors and report (short format in report means we do not query DBS) validate_num('shortformat', param, safe, optional=True) ## validation parameters if not safe.kwargs['workflow'] and safe.kwargs['subresource']: raise InvalidParameter("Invalid input parameters") if safe.kwargs['subresource'] in [ 'data', 'logs' ] and not safe.kwargs['limit'] and not safe.kwargs['jobids']: raise InvalidParameter( "You need to specify the number of jobs to retrieve or their ids." ) elif method in ['DELETE']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False) validate_num("force", param, safe, optional=True) validate_numlist('jobids', param, safe)
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: validate_str("workflow", param, safe, RX_WORKFLOW, optional=False) validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False) validate_str("jobsw", param, safe, RX_CMSSW, optional=False) validate_str("jobarch", param, safe, RX_ARCH, optional=False) jobtype = safe.kwargs.get('jobtype', None) if jobtype == 'Analysis': validate_str("inputdata", param, safe, RX_DATASET, optional=False) validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist']) validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist']) validate_str("splitalgo", param, safe, RX_SPLIT, optional=False) validate_num("algoargs", param, safe, optional=False) validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False) validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False) #validate_str("userisburl", param, safe, re.compile(r"^[A-Za-z]*$"), optional=False) validate_strlist("addoutputfiles", param, safe, RX_ADDFILE) validate_num("savelogsflag", param, safe, optional=False) validate_str("vorole", param, safe, RX_VOPARAMS, optional=True) validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True) validate_str("publishname", param, safe, RX_PUBLISH, optional=True) validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=True) validate_num("publication", param, safe, optional=False) #if publication is set as true both publishDataName and publishDbsUrl are needed if safe.kwargs["publication"] and not (bool(safe.kwargs["publishname"]) and bool(safe.kwargs["publishdbsurl"])): raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication") #if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them) if bool(safe.kwargs["publishname"]) != bool(safe.kwargs["publishdbsurl"]): raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication") validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False) self._checkSite(safe.kwargs['asyncdest']) validate_num("blacklistT1", param, safe, optional=False) validate_str("dbsurl", param, safe, RX_DBSURL, optional=True) validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES) validate_strlist("edmoutfiles", param, safe, RX_OUTFILES) validate_strlist("runs", param, safe, RX_RUNS) validate_strlist("lumis", param, safe, RX_LUMIRANGE) if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]): raise InvalidParameter("The number of runs and the number of lumis lists are different") elif method in ['POST']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False) validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist']) validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist']) elif method in ['GET']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True) validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True) #parameters of subresources calls has to be put here #used by get latest validate_num('age', param, safe, optional=True) #used by get log, get data validate_num('limit', param, safe, optional=True) validate_num('exitcode', param, safe, optional=True) validate_numlist('jobids', param, safe) #used by errors validate_num('shortformat', param, safe, optional=True) #validation parameters if not safe.kwargs['workflow'] and safe.kwargs['subresource']: raise InvalidParameter("Invalid input parameters") if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']: raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.") elif method in ['DELETE']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False) validate_num("force", param, safe, optional=True)
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" authz_login_valid() if method in ['PUT']: validate_str("workflow", param, safe, RX_WORKFLOW, optional=False) validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False) validate_str("jobsw", param, safe, RX_CMSSW, optional=False) validate_str("jobarch", param, safe, RX_ARCH, optional=False) jobtype = safe.kwargs.get('jobtype', None) if jobtype == 'Analysis': validate_str("inputdata", param, safe, RX_DATASET, optional=False) validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites( safe.kwargs['siteblacklist']) validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites( safe.kwargs['sitewhitelist']) validate_str("splitalgo", param, safe, RX_SPLIT, optional=False) validate_num("algoargs", param, safe, optional=False) validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False) validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False) #validate_str("userisburl", param, safe, re.compile(r"^[A-Za-z]*$"), optional=False) validate_strlist("addoutputfiles", param, safe, RX_ADDFILE) validate_num("savelogsflag", param, safe, optional=False) validate_str("vorole", param, safe, RX_VOPARAMS, optional=True) validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True) validate_str("publishname", param, safe, RX_PUBLISH, optional=True) validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=True) validate_num("publication", param, safe, optional=False) #if publication is set as true both publishDataName and publishDbsUrl are needed if safe.kwargs["publication"] and not ( bool(safe.kwargs["publishname"]) and bool(safe.kwargs["publishdbsurl"])): raise InvalidParameter( "You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication" ) #if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them) if bool(safe.kwargs["publishname"]) != bool( safe.kwargs["publishdbsurl"]): raise InvalidParameter( "You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication" ) validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False) self._checkSite(safe.kwargs['asyncdest']) validate_num("blacklistT1", param, safe, optional=False) validate_str("dbsurl", param, safe, RX_DBSURL, optional=True) validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES) validate_strlist("edmoutfiles", param, safe, RX_OUTFILES) validate_strlist("runs", param, safe, RX_RUNS) validate_strlist("lumis", param, safe, RX_LUMIRANGE) if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]): raise InvalidParameter( "The number of runs and the number of lumis lists are different" ) elif method in ['POST']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False) validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites( safe.kwargs['siteblacklist']) validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites( safe.kwargs['sitewhitelist']) elif method in ['GET']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True) validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True) #parameters of subresources calls has to be put here #used by get latest validate_num('age', param, safe, optional=True) #used by get log, get data validate_num('limit', param, safe, optional=True) validate_num('exitcode', param, safe, optional=True) validate_numlist('jobids', param, safe) #used by errors validate_num('shortformat', param, safe, optional=True) #validation parameters if not safe.kwargs['workflow'] and safe.kwargs['subresource']: raise InvalidParameter("Invalid input parameters") if safe.kwargs['subresource'] in [ 'data', 'logs' ] and not safe.kwargs['limit'] and not safe.kwargs['jobids']: raise InvalidParameter( "You need to specify the number of jobs to retrieve or their ids." ) elif method in ['DELETE']: validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False) validate_num("force", param, safe, optional=True)
def validate(self, apiobj, method, api, param, safe): """Validating all the input parameter as enforced by the WMCore.REST module""" if method in ['PUT']: username = cherrypy.request.user['login'] # username registered in SiteDB requestname = param.kwargs['workflow'] param.kwargs['workflow'] = generateTaskName(username, requestname) validate_str("workflow", param, safe, RX_TASKNAME, optional=False) validate_str("activity", param, safe, RX_ACTIVITY, optional=True) validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False) # TODO this should be changed to be non-optional validate_str("generator", param, safe, RX_GENERATOR, optional=True) validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True) validate_str("jobsw", param, safe, RX_CMSSW, optional=False) validate_num("nonprodsw", param, safe, optional=False) validate_str("jobarch", param, safe, RX_ARCH, optional=False) if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw']) validate_num("useparent", param, safe, optional=True) validate_str("secondarydata", param, safe, RX_DATASET, optional=True) validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist']) validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist']) validate_str("splitalgo", param, safe, RX_SPLIT, optional=False) validate_num("algoargs", param, safe, optional=False) validate_num("totalunits", param, safe, optional=True) validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False) validate_str("debugfilename", param, safe, RX_CACHENAME, optional=True) validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False) validate_str("lfn", param, safe, RX_LFN, optional=True) self._checkOutLFN(safe.kwargs, username) validate_strlist("addoutputfiles", param, safe, RX_ADDFILE, custom_err="Incorrect 'JobType.outputFiles' parameter. " \ "Allowed regexp: '%s'." % RX_ADDFILE.pattern) validate_strlist("userfiles", param, safe, RX_USERFILE) validate_num("savelogsflag", param, safe, optional=False) validate_num("saveoutput", param, safe, optional=True) validate_num("faillimit", param, safe, optional=True) validate_num("ignorelocality", param, safe, optional=True) if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []): safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist']) validate_str("vorole", param, safe, RX_VOPARAMS, optional=True) validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True) validate_num("publication", param, safe, optional=False) validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication']))) ## We might want to remove publishname once the backward compatibility ## wont be needed anymore. Then we can just keep publishname2 ## NB: AFAIK the only client not using the CRABLibrary but direct cutl is HC, ## therefore we will need to make sure we do not break it! ## The following two lines will be removed in the future once we will ## not need backward compatibility anymore self._checkPublishDataName(param.kwargs, safe.kwargs['lfn'], requestname, username) validate_str('publishname', param, safe, RX_ANYTHING, optional=True) ##And this if as well, just do self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username) if not safe.kwargs["publishname"]: #new clients won't define this anymore ## The (user specified part of the) publication dataset name must be ## specified and must pass DBS validation. Since this is the correct ## validation function, it must be done before the ## validate_str("publishname", ...) we have below. self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username) else: param.kwargs["publishname2"] = safe.kwargs["publishname"] ## 'publishname' was already validated above in _checkPublishDataName(). ## Calling validate_str with a fake regexp to move the param to the ## list of validated inputs validate_str("publishname2", param, safe, RX_ANYTHING, optional=True) validate_num("publishgroupname", param, safe, optional=True) if safe.kwargs['jobtype'] == 'PrivateMC': if param.kwargs['inputdata']: msg = "Invalid 'inputdata' parameter." msg += " Job type PrivateMC does not take any input dataset." msg += " If you really intend to run over an input dataset, then you must use job type Analysis." raise InvalidParameter(msg) if safe.kwargs['userfiles']: msg = "Invalid 'userfiles' parameter." msg += " Job type PrivateMC does not take any input files." msg += " If you really intend to run over input files, then you must use job type Analysis." raise InvalidParameter(msg) ## Client versions < 3.3.1511 may put in the input dataset something that is not ## really an input dataset (for PrivateMC or user input files). So the only case ## in which we are sure that we have to validate the input dataset is when the ## workflow type is Analysis, the workflow does not run on user input files and ## an input dataset is defined (scriptExe may not define an input). ## Once we don't care anymore about backward compatibility with client < 3.3.1511, ## we can uncomment the 1st line below and delete the next 4 lines. #validate_str("inputdata", param, safe, RX_DATASET, optional=True) if safe.kwargs['jobtype'] == 'Analysis' and not safe.kwargs['userfiles'] and 'inputdata' in param.kwargs: validate_str("inputdata", param, safe, RX_DATASET, optional=True) else: validate_str("inputdata", param, safe, RX_ANYTHING, optional=True) ## The client is not forced to define the primary dataset. So make sure to have ## defaults or take it from the input dataset. The primary dataset is needed for ## the LFN of the output/log files and for publication. We want to have it well ## defined even if publication and/or transfer to storage are off. if safe.kwargs['inputdata']: param.kwargs['primarydataset'] = safe.kwargs['inputdata'].split('/')[1] if not param.kwargs.get('primarydataset', None): if safe.kwargs['jobtype'] == 'PrivateMC': param.kwargs['primarydataset'] = "CRAB_PrivateMC" elif safe.kwargs['jobtype'] == 'Analysis' and safe.kwargs['userfiles']: param.kwargs['primarydataset'] = "CRAB_UserFiles" else: param.kwargs['primarydataset'] = "CRAB_NoInput" ## We validate the primary dataset agains DBS rules even if publication is off, ## because in the future we may want to give the possibility to users to publish ## a posteriori. self._checkPrimaryDataset(param.kwargs, optional=False) validate_str("primarydataset", param, safe, RX_LFNPRIMDS, optional=False) validate_num("nonvaliddata", param, safe, optional=True) #if one and only one between outputDatasetTag and publishDbsUrl is set raise an error (we need both or none of them) validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False) self._checkASODestination(safe.kwargs['asyncdest']) # We no longer use this attribute, but keep it around for older client compatibility validate_num("blacklistT1", param, safe, optional=True) validate_num("oneEventMode", param, safe, optional=True) validate_num("priority", param, safe, optional=True) validate_num("maxjobruntime", param, safe, optional=True) validate_num("numcores", param, safe, optional=True) validate_num("maxmemory", param, safe, optional=True) validate_str("dbsurl", param, safe, RX_DBSURL, optional=False) validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect tfileoutfiles parameter (TFileService). " \ "Allowed regexp: '%s'." % RX_OUTFILES.pattern) validate_strlist("edmoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect edmoutfiles parameter (PoolOutputModule). " \ "Allowed regexp: '%s'." % RX_OUTFILES.pattern) validate_strlist("runs", param, safe, RX_RUNS) validate_strlist("lumis", param, safe, RX_LUMIRANGE) #validate_str("scheduler", param, safe, RX_SCHEDULER) if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]): raise InvalidParameter("The number of runs and the number of lumis lists are different") validate_strlist("adduserfiles", param, safe, RX_ADDFILE) validate_str("asourl", param, safe, RX_ASOURL, optional=True) validate_str("asodb", param, safe, RX_ASODB, optional=True) safe.kwargs["asourl"], safe.kwargs["asodb"] = self._getAsoConfig(safe.kwargs["asourl"], safe.kwargs["asodb"]) validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True) validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS) validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True) validate_str("collector", param, safe, RX_COLLECTOR, optional=True) validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS) validate_num("dryrun", param, safe, optional=True) validate_num("ignoreglobalblacklist", param, safe, optional=True) elif method in ['POST']: validate_str("workflow", param, safe, RX_TASKNAME, optional=False) validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True) validate_numlist('jobids', param, safe) ## In a resubmission, the site black- and whitelists need to be interpreted ## differently than in an initial task submission. If there is no site black- ## or whitelist, set it to None and DataWorkflow will use the corresponding ## list defined in the initial task submission. If the site black- or whitelist ## is equal to the string 'empty', set it to an empty list and don't call ## validate_strlist as it would fail. if 'siteblacklist' not in param.kwargs: safe.kwargs['siteblacklist'] = None elif param.kwargs['siteblacklist'] == 'empty': safe.kwargs['siteblacklist'] = [] del param.kwargs['siteblacklist'] else: validate_strlist("siteblacklist", param, safe, RX_CMSSITE) safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist']) if 'sitewhitelist' not in param.kwargs: safe.kwargs['sitewhitelist'] = None elif param.kwargs['sitewhitelist'] == 'empty': safe.kwargs['sitewhitelist'] = [] del param.kwargs['sitewhitelist'] else: validate_strlist("sitewhitelist", param, safe, RX_CMSSITE) safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist']) validate_num("maxjobruntime", param, safe, optional=True) validate_num("maxmemory", param, safe, optional=True) validate_num("numcores", param, safe, optional=True) validate_num("priority", param, safe, optional=True) validate_num("force", param, safe, optional=True) validate_num("publication", param, safe, optional=True) elif method in ['GET']: validate_str("workflow", param, safe, RX_TASKNAME, optional=True) validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True) validate_str('username', param, safe, RX_USERNAME, optional=True) validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric ## Used to determine how much information to return to the client for status. ## also used by report to determine if it has to check job states validate_num("verbose", param, safe, optional=True) ## used by get log, get data validate_num('limit', param, safe, optional=True) validate_num('exitcode', param, safe, optional=True) validate_numlist('jobids', param, safe) ## used by errors and report (short format in report means we do not query DBS) validate_num('shortformat', param, safe, optional=True) ## validation parameters if not safe.kwargs['workflow'] and safe.kwargs['subresource']: raise InvalidParameter("Invalid input parameters") if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']: raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.") elif method in ['DELETE']: validate_str("workflow", param, safe, RX_TASKNAME, optional=False) validate_num("force", param, safe, optional=True) validate_numlist('jobids', param, safe) validate_str("killwarning", param, safe, RX_TEXT_FAIL, optional=True) #decode killwarning message if present if safe.kwargs['killwarning']: try: safe.kwargs['killwarning'] = b64decode(safe.kwargs['killwarning']) except TypeError: raise InvalidParameter("Failure message is not in the accepted format")
def validate(self, apiobj, method, api, param, safe): #pylint: disable=unused-argument """Validating all the input parameter as enforced by the WMCore.REST module""" # TODO, use authz_user_action() which will allow only user to maintain and modify its own # documents authz_login_valid() if method in ['PUT']: # Do we want to validate everything? # Now what is put in CouchDB is not validated # And all put are prepared by us in JOB wrappers, so it should already be correct. # P.S. Validation is done in function and it double check if all required keys are available print (param, safe) validate_str("id", param, safe, RX_ANYTHING, optional=False) validate_str("username", param, safe, RX_ANYTHING, optional=False) validate_str("taskname", param, safe, RX_ANYTHING, optional=False) validate_str("destination", param, safe, RX_ANYTHING, optional=False) validate_str("destination_lfn", param, safe, RX_ANYTHING, optional=False) validate_str("source", param, safe, RX_ANYTHING, optional=False) validate_str("source_lfn", param, safe, RX_ANYTHING, optional=False) validate_num("filesize", param, safe, optional=False) validate_num("publish", param, safe, optional=False) validate_str("job_id", param, safe, RX_JOBID, optional=False) validate_num("job_retry_count", param, safe, optional=False) validate_str("type", param, safe, RX_ANYTHING, optional=False) validate_str("asoworker", param, safe, RX_ANYTHING, optional=True) validate_num("transfer_retry_count", param, safe, optional=True) validate_num("transfer_max_retry_count", param, safe, optional=True) validate_num("publication_retry_count", param, safe, optional=True) validate_num("publication_max_retry_count", param, safe, optional=True) validate_num("start_time", param, safe, optional=False) validate_str("rest_host", param, safe, RX_ANYTHING, optional=False) validate_str("rest_uri", param, safe, RX_ANYTHING, optional=False) validate_str("transfer_state", param, safe, RX_ANYTHING, optional=False) validate_str("publication_state", param, safe, RX_ANYTHING, optional=False) validate_str("fts_id", param, safe, RX_ANYTHING, optional=True) validate_str("fts_instance", param, safe, RX_ANYTHING, optional=True) if method in ['POST']: # POST is for update, so we should allow anyone anything? # Of Course no, but there are multiple combinations, so we are not validating here # and all validation is in post function validate_str("subresource", param, safe, RX_SUBPOSTUSERTRANSFER, optional=False) validate_str("id", param, safe, RX_ANYTHING, optional=True) validate_str("username", param, safe, RX_USERNAME, optional=True) validate_str("taskname", param, safe, RX_TASKNAME, optional=True) validate_num("start_time", param, safe, optional=True) validate_str("source", param, safe, RX_ANYTHING, optional=True) validate_str("source_lfn", param, safe, RX_ANYTHING, optional=True) validate_num("filesize", param, safe, optional=True) validate_str("transfer_state", param, safe, RX_ANYTHING, optional=True) validate_num("transfer_retry_count", param, safe, optional=True) validate_str("publication_state", param, safe, RX_ANYTHING, optional=True) validate_str("job_id", param, safe, RX_JOBID, optional=True) validate_num("job_retry_count", param, safe, optional=True) validate_strlist("listOfIds", param, safe, RX_ANYTHING) # Interesting... TODO. Have optional in strlist elif method in ['GET']: validate_str("subresource", param, safe, RX_SUBGETUSERTRANSFER, optional=False) validate_str("id", param, safe, RX_TASKNAME, optional=True) validate_str("username", param, safe, RX_USERNAME, optional=True) validate_str("taskname", param, safe, RX_TASKNAME, optional=True) elif method in ['DELETE']: raise UnsupportedMethod('This method is not supported in this API!')