Ejemplo n.º 1
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid()

        if method in ['PUT']:
            #TODO check optional parameter
            #TODO check all the regexp
            validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
            validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
            validate_numlist("outfileruns", param, safe)
            if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]):
                raise InvalidParameter("The number of runs and the number of lumis lists are different")
            validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
            validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
            validate_num("pandajobid", param, safe, optional=False)
            validate_num("outsize", param, safe, optional=False)
            validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False)
            validate_str("appver", param, safe, RX_CMSSW, optional=False)
            validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
            validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False)
            validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False)
            validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False)
            validate_str("outlocation", param, safe, RX_CMSSITE, optional=False)
            validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False)
            validate_str("acquisitionera", param, safe, RX_WORKFLOW, optional=False)#TODO Do we really need this?
            validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex
            validate_str("outlfn", param, safe, RX_LFN, optional=False)
            validate_num("events", param, safe, optional=False)
        elif method in ['POST']:
            raise NotImplementedError
        elif method in ['GET']:
            validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
            validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
        elif method in ['DELETE']:
            raise NotImplementedError
Ejemplo n.º 2
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid()  # TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here.
        #      Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource)

        if method in ["POST"]:
            validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
            validate_str("status", param, safe, RX_STATUS, optional=True)
            validate_str("command", param, safe, RX_STATUS, optional=True)
            validate_str("getstatus", param, safe, RX_STATUS, optional=True)
            validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
            validate_numlist("resubmittedjobs", param, safe)
            validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
            validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
            validate_num("limit", param, safe, optional=True)
            validate_num("clusterid", param, safe, optional=True)  # clusterid of the dag
            # possible combinations to check
            # 1) taskname + status
            # 2) taskname + status + failure
            # 3) taskname + status + resubmitted + jobsetid
            # 4) taskname + status == (1)
            # 5)            status + limit + getstatus + workername
            # 6) taskname + runs + lumis
        elif method in ["GET"]:
            validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
            validate_str("getstatus", param, safe, RX_STATUS, optional=True)
            validate_num("limit", param, safe, optional=True)
            validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
Ejemplo n.º 3
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid()

        if method in ['PUT']:
            #TODO check optional parameter
            #TODO check all the regexp
            validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
            validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
            validate_numlist("outfileruns", param, safe)
            if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]):
                raise InvalidParameter("The number of runs and the number of lumis lists are different")
            validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
            validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
            validate_str("jobid", param, safe, RX_JOBID, optional=True)
            #TODO: for backward compatibility. Get rid of the pandajobid once all jobs using it are done (one month after this gets released)
            #(The following five lines can be deleted)
            validate_num("pandajobid", param, safe, optional=True)
            if bool(safe.kwargs["jobid"]) == bool(safe.kwargs["pandajobid"]):
                raise InvalidParameter("Only one among jobid and pandajobid should be set")
            #Oracle/cx_oracle/python stack does not like None for numbers, even if they are nullable
            if safe.kwargs["pandajobid"] == None:
                safe.kwargs["pandajobid"] =  0
            validate_num("outsize", param, safe, optional=False)
            validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False)
            validate_str("appver", param, safe, RX_CMSSW, optional=False)
            validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
            validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False)
            validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False)
            validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False)
            validate_str("outlocation", param, safe, RX_CMSSITE, optional=False)
            validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False)
            validate_str("acquisitionera", param, safe, RX_TASKNAME, optional=False)#TODO Do we really need this?
            validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex
            validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
            validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
            validate_num("events", param, safe, optional=False)
            validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
            validate_num("directstageout", param, safe, optional=True)
            safe.kwargs["directstageout"] = 'T' if safe.kwargs["directstageout"] else 'F' #'F' if not provided
        elif method in ['POST']:
            validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
            validate_str("outlfn", param, safe, RX_LFN, optional=False)
            validate_str("filestate", param, safe, RX_FILESTATE, optional=False)
        elif method in ['GET']:
            validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
            validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
            validate_num("howmany", param, safe, optional=True)
        elif method in ['DELETE']:
            authz_operator()
            validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
            validate_str("hours", param, safe, RX_HOURS, optional=True)
            if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
               raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
                                        " will be deleted. Only one of the two parameters can be specified.")
Ejemplo n.º 4
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid()

        if method in ['PUT']:
            #TODO check optional parameter
            #TODO check all the regexp
            validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
            validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
            validate_numlist("outfileruns", param, safe)
            if len(safe.kwargs["outfileruns"]) != len(safe.kwargs["outfilelumis"]):
                raise InvalidParameter("The number of runs and the number of lumis lists are different")
            validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
            validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
            validate_num("pandajobid", param, safe, optional=False)
            validate_num("outsize", param, safe, optional=False)
            validate_str("publishdataname", param, safe, RX_PUBLISH, optional=False)
            validate_str("appver", param, safe, RX_CMSSW, optional=False)
            validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
            validate_str("checksummd5", param, safe, RX_CHECKSUM, optional=False)
            validate_str("checksumcksum", param, safe, RX_CHECKSUM, optional=False)
            validate_str("checksumadler32", param, safe, RX_CHECKSUM, optional=False)
            validate_str("outlocation", param, safe, RX_CMSSITE, optional=False)
            validate_str("outtmplocation", param, safe, RX_CMSSITE, optional=False)
            validate_str("acquisitionera", param, safe, RX_WORKFLOW, optional=False)#TODO Do we really need this?
            validate_str("outdatasetname", param, safe, RX_OUTDSLFN, optional=False)#TODO temporary, need to come up with a regex
            validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
            validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
            validate_num("events", param, safe, optional=False)
            validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
            validate_num("directstageout", param, safe, optional=True)
            safe.kwargs["directstageout"] = 'T' if safe.kwargs["directstageout"] else 'F' #'F' if not provided
        elif method in ['POST']:
            validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
            validate_str("outlfn", param, safe, RX_LFN, optional=False)
            validate_str("filestate", param, safe, RX_FILESTATE, optional=False)
        elif method in ['GET']:
            validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
            validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
        elif method in ['DELETE']:
            authz_operator()
            validate_str("taskname", param, safe, RX_WORKFLOW, optional=True)
            validate_str("hours", param, safe, RX_HOURS, optional=True)
            if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
               raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
                                        " will be deleted. Only one of the two parameters can be specified.")
Ejemplo n.º 5
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid() #TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here.
                            #      Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource)

        if method in ['PUT']:
            validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
            validate_num("subjobdef", param, safe, optional=True)
            validate_str("substatus", param, safe, RX_STATUS, optional=False)
            validate_strlist("subblocks", param, safe, RX_BLOCK)
            validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True)
            validate_str("subuser", param, safe, RX_DN, optional=False)
        elif method in ['POST']:
            validate_str("workflow", param, safe, RX_WORKFLOW, optional=True)
            validate_str("status", param, safe, RX_STATUS, optional=True)
            validate_str("getstatus", param, safe, RX_STATUS, optional=True)
            validate_num("jobset", param, safe, optional=True)
            validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
            validate_numlist("resubmittedjobs", param, safe)
            validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
            validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
            validate_num("limit", param, safe, optional=True)
            validate_strlist("runs", param, safe, RX_RUNS)
            validate_strlist("lumis", param, safe, RX_LUMIRANGE)
            # possible combinations to check
            # 1) taskname + status
            # 2) taskname + status + failure
            # 3) taskname + status + resubmitted + jobsetid
            # 4) taskname + status == (1)
            # 5)            status + limit + getstatus + workername
            # 6) taskname + runs + lumis
        elif method in ['GET']:
            validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
            validate_str("getstatus", param, safe, RX_STATUS, optional=True)
            validate_num("limit", param, safe, optional=True)
            validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
            validate_num("subjobdef", param, safe, optional=True)
            validate_str("subuser", param, safe, RX_DN, optional=True)
            # possible combinations to check
            # 1) workername + getstatus + limit
            # 2) subresource + subjobdef + subuser
        elif method in ['DELETE']:
            pass
Ejemplo n.º 6
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid()

        if method in ['PUT']:
            validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
            validate_num("subjobdef", param, safe, optional=True)
            validate_str("substatus", param, safe, RX_STATUS, optional=False)
            validate_strlist("subblocks", param, safe, RX_BLOCK)
            validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True)
            validate_str("subuser", param, safe, RX_DN, optional=False)
        elif method in ['POST']:
            validate_str("workflow", param, safe, RX_WORKFLOW, optional=True)
            validate_str("status", param, safe, RX_STATUS, optional=True)
            validate_str("getstatus", param, safe, RX_STATUS, optional=True)
            validate_num("jobset", param, safe, optional=True)
            validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
            validate_numlist("resubmittedjobs", param, safe)
            validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
            validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
            validate_num("limit", param, safe, optional=True)
            validate_strlist("runs", param, safe, RX_RUNS)
            validate_strlist("lumis", param, safe, RX_LUMIRANGE)
            validate_strlist("outputdataset", param, safe, RX_OUT_DATASET)
            # possible combinations to check
            # 1) taskname + status
            # 2) taskname + status + failure
            # 3) taskname + status + resubmitted + jobsetid
            # 4) taskname + status == (1)
            # 5)            status + limit + getstatus + workername
            # 6) taskname + runs + lumis
        elif method in ['GET']:
            validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
            validate_str("getstatus", param, safe, RX_STATUS, optional=True)
            validate_num("limit", param, safe, optional=True)
            validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
            validate_num("subjobdef", param, safe, optional=True)
            validate_str("subuser", param, safe, RX_DN, optional=True)
            # possible combinations to check
            # 1) workername + getstatus + limit
            # 2) subresource + subjobdef + subuser
        elif method in ['DELETE']:
            pass
Ejemplo n.º 7
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid()

        if method in ['PUT']:
            validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
            validate_num("subjobdef", param, safe, optional=True)
            validate_str("substatus", param, safe, RX_STATUS, optional=False)
            validate_strlist("subblocks", param, safe, RX_BLOCK)
            validate_str("subfailure", param, safe, RX_TEXT_FAIL, optional=True)
            validate_str("subuser", param, safe, RX_DN, optional=False)
        elif method in ['POST']:
            validate_str("workflow", param, safe, RX_WORKFLOW, optional=True)
            validate_str("status", param, safe, RX_STATUS, optional=True)
            validate_str("getstatus", param, safe, RX_STATUS, optional=True)
            validate_num("jobset", param, safe, optional=True)
            validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
            validate_numlist("resubmittedjobs", param, safe)
            validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
            validate_str("subresource", param, safe, RX_SUBPOSTWORKER, optional=True)
            validate_num("limit", param, safe, optional=True)
            validate_strlist("runs", param, safe, RX_RUNS)
            validate_strlist("lumis", param, safe, RX_LUMIRANGE)
            # possible combinations to check
            # 1) taskname + status
            # 2) taskname + status + failure
            # 3) taskname + status + resubmitted + jobsetid
            # 4) taskname + status == (1)
            # 5)            status + limit + getstatus + workername
            # 6) taskname + runs + lumis
        elif method in ['GET']:
            validate_str("workername", param, safe, RX_WORKER_NAME, optional=True)
            validate_str("getstatus", param, safe, RX_STATUS, optional=True)
            validate_num("limit", param, safe, optional=True)
            validate_str("subresource", param, safe, RX_SUBGETWORKER, optional=True)
            validate_num("subjobdef", param, safe, optional=True)
            validate_str("subuser", param, safe, RX_DN, optional=True)
            # possible combinations to check
            # 1) workername + getstatus + limit
            # 2) subresource + subjobdef + subuser
        elif method in ['DELETE']:
            pass
Ejemplo n.º 8
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid()

        if method in ['PUT']:
            validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
            validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
            validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
            validate_str("jobarch", param, safe, RX_ARCH, optional=False)
            jobtype = safe.kwargs.get('jobtype', None)
            if jobtype == 'Analysis':
                validate_str("inputdata", param, safe, RX_DATASET, optional=False)
            validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
            safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
            validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
            safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
            validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
            validate_num("algoargs", param, safe, optional=False)
            validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
            validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
            #validate_str("userisburl", param, safe, re.compile(r"^[A-Za-z]*$"), optional=False)
            validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
            validate_num("savelogsflag", param, safe, optional=False)
            validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
            validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
            validate_str("publishname", param, safe, RX_PUBLISH, optional=True)
            validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=True)
            validate_num("publication", param, safe, optional=False)
            #if publication is set as true both publishDataName and publishDbsUrl are needed
            if safe.kwargs["publication"] and not (bool(safe.kwargs["publishname"]) and bool(safe.kwargs["publishdbsurl"])):
                raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication")
            #if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
            if bool(safe.kwargs["publishname"]) != bool(safe.kwargs["publishdbsurl"]):
                raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication")
            validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
            self._checkSite(safe.kwargs['asyncdest'])
            validate_num("blacklistT1", param, safe, optional=False)
            validate_str("dbsurl", param, safe, RX_DBSURL, optional=True)
            validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("runs", param, safe, RX_RUNS)
            validate_strlist("lumis", param, safe, RX_LUMIRANGE)
            if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
                raise InvalidParameter("The number of runs and the number of lumis lists are different")

        elif method in ['POST']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
            validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
            safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
            validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
            safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])

        elif method in ['GET']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
            validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)

            #parameters of subresources calls has to be put here
            #used by get latest
            validate_num('age', param, safe, optional=True)

            #used by get log, get data
            validate_num('limit', param, safe, optional=True)
            validate_num('exitcode', param, safe, optional=True)
            validate_numlist('jobids', param, safe)

            #used by errors
            validate_num('shortformat', param, safe, optional=True)

            #validation parameters
            if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
                raise InvalidParameter("Invalid input parameters")
            if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
                raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")

        elif method in ['DELETE']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
            validate_num("force", param, safe, optional=True)
Ejemplo n.º 9
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        #authz_login_valid()

        if method in ['PUT']:
            validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
            validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
            # TODO this should be changed to be non-optional
            validate_str("generator", param, safe, RX_GENERATOR, optional=True)
            validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
            validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
            validate_num("nonprodsw", param, safe, optional=False)
            validate_str("jobarch", param, safe, RX_ARCH, optional=False)
            if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
                self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
            jobtype = safe.kwargs.get('jobtype', None)
            validate_num("useparent", param, safe, optional=True)
            validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
            safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
            validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
            safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
            validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
            validate_num("algoargs", param, safe, optional=False)
            validate_num("totalunits", param, safe, optional=True)
            validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
            validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
            validate_str("lfn", param, safe, RX_LFN, optional=True)
            self._checkOutLFN(safe.kwargs)
            validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
            validate_strlist("userfiles", param, safe, RX_USERFILE)
            validate_num("savelogsflag", param, safe, optional=False)
            validate_num("saveoutput", param, safe, optional=True)
            validate_num("faillimit", param, safe, optional=True)
            validate_num("ignorelocality", param, safe, optional=True)
            if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
                safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
            validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
            validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
            validate_num("publication", param, safe, optional=False)
            validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication'])))
            ## Validations needed in case publication is on.
            if safe.kwargs['publication']:
                ## The (user specified part of the) publication dataset name must be
                ## specified and must pass DBS validation. Since this is the correct
                ## validation function, it must be done before the
                ## validate_str("workflow", ...) and validate_str("publishname", ...)
                ## we have below. Not sure why RX_PUBLISH was introduced in CRAB, but
                ## it is not the correct regular expression for publication dataset
                ## name validation.
                self._checkPublishDataName(param.kwargs, safe.kwargs['lfn'])
                ## 'publishname' was already validated above in _checkPublishDataName().
                ## But I am not sure if we can skip the validate_str('publishname', ...)
                ## or we need it so that all parameters are moved from param to safe.
                ## Therefore, I didn't remove the validate_str('publishname', ...),
                ## but only changed RX_PUBLISH -> RX_ANYTHING.
                validate_str('publishname', param, safe, RX_ANYTHING, optional=False)
                ## If this is a MC generation job type, the primary dataset name will
                ## be used to define the primary dataset field in the publication
                ## dataset name. Therefore, the primary dataset name must pass DBS
                ## validation. Since in a MC generation job type the primary dataset
                ## name is saved in 'inputdata', _checkPrimaryDataset() actually
                ## validates 'inputdata'. Therefore, this function must come before
                ## the validate_str('inputdata', ...) we have below. OTOH, matching
                ## 'inputdata' agains RX_DATASET is not the correct validation for
                ## the primary dataset name.
                if jobtype == 'PrivateMC':
                    self._checkPrimaryDataset(param.kwargs)
            else:
                ## Not sure if we need to match publishname against RX_PUBLISH...
                validate_str("publishname", param, safe, RX_PUBLISH, optional=False)
            validate_num("publishgroupname", param, safe, optional=True)
            ## This line must come after _checkPublishDataName()
            validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
            if jobtype == 'Analysis':
                validate_str("inputdata", param, safe, RX_DATASET, optional=False)
            ## Added this if, because in this case 'inputdata' was already validated
            ## above in _checkPrimaryDataset(), and I am not sure if we can skip the
            ## validate_str('inputdata', ...) or we need it so that all parameters
            ## are moved from param to safe.
            elif jobtype == 'PrivateMC' and safe.kwargs['publication']:
                validate_str("inputdata", param, safe, RX_ANYTHING, optional=False)
            else:
                validate_str("inputdata", param, safe, RX_DATASET, optional=True)
            validate_num("nonvaliddata", param, safe, optional=True)
            #if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
            validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
            self._checkASODestination(safe.kwargs['asyncdest'])
            # We no longer use this attribute, but keep it around for older client compatibility
            validate_num("blacklistT1", param, safe, optional=True)
            validate_num("oneEventMode", param, safe, optional=True)
            validate_num("priority", param, safe, optional=True)
            validate_num("maxjobruntime", param, safe, optional=True)
            validate_num("numcores", param, safe, optional=True)
            validate_num("maxmemory", param, safe, optional=True)
            validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
            validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("runs", param, safe, RX_RUNS)
            validate_strlist("lumis", param, safe, RX_LUMIRANGE)
            #validate_str("scheduler", param, safe, RX_SCHEDULER)
            if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
                raise InvalidParameter("The number of runs and the number of lumis lists are different")
            validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
            validate_str("asourl", param, safe, RX_ASOURL, optional=True)
            validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
            validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
            validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
            validate_str("collector", param, safe, RX_COLLECTOR, optional=True)
            validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
            validate_num("dryrun", param, safe, optional=True)

        elif method in ['POST']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
            validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True)
            ## In a resubmission, the site black- and whitelists need to be interpreted
            ## differently than in an initial task submission. If there is no site black-
            ## or whitelist, set it to None and DataWorkflow will use the corresponding
            ## list defined in the initial task submission. If the site black- or whitelist
            ## is equal to the string 'empty', set it to an empty list and don't call
            ## validate_strlist as it would fail.
            if 'siteblacklist' not in param.kwargs:
                safe.kwargs['siteblacklist'] = None
            elif param.kwargs['siteblacklist'] == 'empty':
                safe.kwargs['siteblacklist'] = []
                del param.kwargs['siteblacklist']
            else:
                validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
                safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
            if 'sitewhitelist' not in param.kwargs:
                safe.kwargs['sitewhitelist'] = None
            elif param.kwargs['sitewhitelist'] == 'empty':
                safe.kwargs['sitewhitelist'] = []
                del param.kwargs['sitewhitelist']
            else:
                validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
                safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
            validate_numlist('jobids', param, safe)
            validate_num("priority", param, safe, optional=True)
            validate_num("maxjobruntime", param, safe, optional=True)
            validate_num("numcores", param, safe, optional=True)
            validate_num("maxmemory", param, safe, optional=True)
            validate_num("force", param, safe, optional=True)

        elif method in ['GET']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
            validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
            validate_str('username', param, safe, RX_USERNAME, optional=True)
            validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric

            ## Used to determine how much information to return to the client for status.
            ## also used by report to determine if it has to check job states
            validate_num("verbose", param, safe, optional=True)

            ## used by get log, get data
            validate_num('limit', param, safe, optional=True)
            validate_num('exitcode', param, safe, optional=True)
            validate_numlist('jobids', param, safe)

            ## used by errors and report (short format in report means we do not query DBS)
            validate_num('shortformat', param, safe, optional=True)

            ## validation parameters
            if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
                raise InvalidParameter("Invalid input parameters")
            if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
                raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")

        elif method in ['DELETE']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
            validate_num("force", param, safe, optional=True)
            validate_numlist('jobids', param, safe)
Ejemplo n.º 10
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid(
        )  #TODO: should we also call authz_operator here ? Otherwise anybody can get tasks from here.
        #      Actually, maybe something even more strict is necessary (only the prod TW machine can access this resource)

        if method in ['PUT']:
            validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
            validate_num("subjobdef", param, safe, optional=True)
            validate_str("substatus", param, safe, RX_STATUS, optional=False)
            validate_strlist("subblocks", param, safe, RX_BLOCK)
            validate_str("subfailure",
                         param,
                         safe,
                         RX_TEXT_FAIL,
                         optional=True)
            validate_str("subuser", param, safe, RX_DN, optional=False)
        elif method in ['POST']:
            validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
            validate_str("status", param, safe, RX_STATUS, optional=True)
            validate_str("getstatus", param, safe, RX_STATUS, optional=True)
            validate_num("jobset", param, safe, optional=True)
            validate_str("failure", param, safe, RX_TEXT_FAIL, optional=True)
            validate_numlist("resubmittedjobs", param, safe)
            validate_str("workername",
                         param,
                         safe,
                         RX_WORKER_NAME,
                         optional=True)
            validate_str("subresource",
                         param,
                         safe,
                         RX_SUBPOSTWORKER,
                         optional=True)
            validate_num("limit", param, safe, optional=True)
            validate_strlist("runs", param, safe, RX_RUNS)
            validate_strlist("lumis", param, safe, RX_LUMIRANGE)
            # possible combinations to check
            # 1) taskname + status
            # 2) taskname + status + failure
            # 3) taskname + status + resubmitted + jobsetid
            # 4) taskname + status == (1)
            # 5)            status + limit + getstatus + workername
            # 6) taskname + runs + lumis
        elif method in ['GET']:
            validate_str("workername",
                         param,
                         safe,
                         RX_WORKER_NAME,
                         optional=True)
            validate_str("getstatus", param, safe, RX_STATUS, optional=True)
            validate_num("limit", param, safe, optional=True)
            validate_str("subresource",
                         param,
                         safe,
                         RX_SUBGETWORKER,
                         optional=True)
            validate_num("subjobdef", param, safe, optional=True)
            validate_str("subuser", param, safe, RX_DN, optional=True)
            # possible combinations to check
            # 1) workername + getstatus + limit
            # 2) subresource + subjobdef + subuser
        elif method in ['DELETE']:
            pass
Ejemplo n.º 11
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        #authz_login_valid()

        if method in ['PUT']:
            validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
            validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
            validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
            validate_str("generator", param, safe, RX_GENERATOR, optional=True)
            validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
            validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
            validate_num("nonprodsw", param, safe, optional=False)
            validate_str("jobarch", param, safe, RX_ARCH, optional=False)
            if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
                self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
            jobtype = safe.kwargs.get('jobtype', None)
            if jobtype == 'Analysis':
                validate_str("inputdata", param, safe, RX_DATASET, optional=False)
            else:
                validate_str("inputdata", param, safe, RX_DATASET, optional=True)
            validate_num("useparent", param, safe, optional=True)
            validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
            safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
            validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
            safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
            validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
            validate_num("algoargs", param, safe, optional=False)
            validate_num("totalunits", param, safe, optional=True)
            validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
            validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
            validate_str("lfnprefix", param, safe, RX_LFNPATH, optional=True) #keeping this for compatibility with old clients. Will be converted into a full lfn
            validate_str("lfn", param, safe, RX_LFN, optional=True)
            self._checkOutLFN(safe.kwargs)
            validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
            validate_strlist("userfiles", param, safe, RX_USERFILE)
            validate_num("savelogsflag", param, safe, optional=False)
            validate_num("saveoutput", param, safe, optional=True)
            validate_num("faillimit", param, safe, optional=True)
            validate_num("ignorelocality", param, safe, optional=True)
            if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
                safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
            validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
            validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
            validate_str("publishname", param, safe, RX_PUBLISH, optional=False)
            validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=True)
            validate_num("publication", param, safe, optional=False)
            #if publication is set as true both publishDataName and publishDbsUrl are needed
            if safe.kwargs["publication"] and not (bool(safe.kwargs["publishname"]) and bool(safe.kwargs["publishdbsurl"])):
                raise InvalidParameter("You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication")
            #if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
            validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
            self._checkASODestination(safe.kwargs['asyncdest'])
            # We no longer use this attribute, but keep it around for older client compatibility
            validate_num("blacklistT1", param, safe, optional=True)
            validate_num("oneEventMode", param, safe, optional=True)
            validate_num("priority", param, safe, optional=True)
            validate_num("maxjobruntime", param, safe, optional=True)
            validate_num("numcores", param, safe, optional=True)
            validate_num("maxmemory", param, safe, optional=True)
            validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
            validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("runs", param, safe, RX_RUNS)
            validate_strlist("lumis", param, safe, RX_LUMIRANGE)
            #validate_str("scheduler", param, safe, RX_SCHEDULER)
            if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
                raise InvalidParameter("The number of runs and the number of lumis lists are different")
            validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
            validate_str("asourl", param, safe, RX_ASOURL, optional=True)
            validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
            validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
            validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
            validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)

        elif method in ['POST']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
            validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
            safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
            validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
            safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
            validate_numlist('jobids', param, safe)
            validate_num("priority", param, safe, optional=True)
            validate_num("maxjobruntime", param, safe, optional=True)
            validate_num("numcores", param, safe, optional=True)
            validate_num("maxmemory", param, safe, optional=True)


        elif method in ['GET']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
            validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
            validate_str('username', param, safe, RX_USERNAME, optional=True)
            validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric

            ## Used to determine how much information to return to the client for status.
            ## also used by report to determine if it has to check job states
            validate_num("verbose", param, safe, optional=True)

            ## used by get log, get data
            validate_num('limit', param, safe, optional=True)
            validate_num('exitcode', param, safe, optional=True)
            validate_numlist('jobids', param, safe)

            ## used by errors and report (short format in report means we do not query DBS)
            validate_num('shortformat', param, safe, optional=True)

            ## validation parameters
            if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
                raise InvalidParameter("Invalid input parameters")
            if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
                raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")

        elif method in ['DELETE']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
            validate_num("force", param, safe, optional=True)
            validate_numlist('jobids', param, safe)
Ejemplo n.º 12
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid()

        if method in ['PUT']:
            validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
            validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
            validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
            validate_str("jobarch", param, safe, RX_ARCH, optional=False)
            jobtype = safe.kwargs.get('jobtype', None)
            if jobtype == 'Analysis':
                validate_str("inputdata",
                             param,
                             safe,
                             RX_DATASET,
                             optional=False)
            validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
            safe.kwargs['siteblacklist'] = self._expandSites(
                safe.kwargs['siteblacklist'])
            validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
            safe.kwargs['sitewhitelist'] = self._expandSites(
                safe.kwargs['sitewhitelist'])
            validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
            validate_num("algoargs", param, safe, optional=False)
            validate_str("cachefilename",
                         param,
                         safe,
                         RX_CACHENAME,
                         optional=False)
            validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
            #validate_str("userisburl", param, safe, re.compile(r"^[A-Za-z]*$"), optional=False)
            validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
            validate_num("savelogsflag", param, safe, optional=False)
            validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
            validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
            validate_str("publishname", param, safe, RX_PUBLISH, optional=True)
            validate_str("publishdbsurl",
                         param,
                         safe,
                         RX_DBSURL,
                         optional=True)
            validate_num("publication", param, safe, optional=False)
            #if publication is set as true both publishDataName and publishDbsUrl are needed
            if safe.kwargs["publication"] and not (
                    bool(safe.kwargs["publishname"])
                    and bool(safe.kwargs["publishdbsurl"])):
                raise InvalidParameter(
                    "You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication"
                )
            #if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
            if bool(safe.kwargs["publishname"]) != bool(
                    safe.kwargs["publishdbsurl"]):
                raise InvalidParameter(
                    "You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication"
                )
            validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
            self._checkSite(safe.kwargs['asyncdest'])
            validate_num("blacklistT1", param, safe, optional=False)
            validate_str("dbsurl", param, safe, RX_DBSURL, optional=True)
            validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("runs", param, safe, RX_RUNS)
            validate_strlist("lumis", param, safe, RX_LUMIRANGE)
            if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
                raise InvalidParameter(
                    "The number of runs and the number of lumis lists are different"
                )

        elif method in ['POST']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
            validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
            safe.kwargs['siteblacklist'] = self._expandSites(
                safe.kwargs['siteblacklist'])
            validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
            safe.kwargs['sitewhitelist'] = self._expandSites(
                safe.kwargs['sitewhitelist'])

        elif method in ['GET']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
            validate_str('subresource',
                         param,
                         safe,
                         RX_SUBRESTAT,
                         optional=True)

            #parameters of subresources calls has to be put here
            #used by get latest
            validate_num('age', param, safe, optional=True)

            #used by get log, get data
            validate_num('limit', param, safe, optional=True)
            validate_num('exitcode', param, safe, optional=True)
            validate_numlist('jobids', param, safe)

            #used by errors
            validate_num('shortformat', param, safe, optional=True)

            #validation parameters
            if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
                raise InvalidParameter("Invalid input parameters")
            if safe.kwargs['subresource'] in [
                    'data', 'logs'
            ] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
                raise InvalidParameter(
                    "You need to specify the number of jobs to retrieve or their ids."
                )

        elif method in ['DELETE']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
            validate_num("force", param, safe, optional=True)
Ejemplo n.º 13
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        #authz_login_valid()

        if method in ['PUT']:
            validate_str("workflow", param, safe, RX_WORKFLOW, optional=False)
            validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
            validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
            validate_str("generator", param, safe, RX_GENERATOR, optional=True)
            validate_str("eventsperlumi",
                         param,
                         safe,
                         RX_LUMIEVENTS,
                         optional=True)
            validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
            validate_num("nonprodsw", param, safe, optional=False)
            validate_str("jobarch", param, safe, RX_ARCH, optional=False)
            if not safe.kwargs[
                    "nonprodsw"]:  #if the user wants to allow non-production releases
                self._checkReleases(safe.kwargs['jobarch'],
                                    safe.kwargs['jobsw'])
            jobtype = safe.kwargs.get('jobtype', None)
            if jobtype == 'Analysis':
                validate_str("inputdata",
                             param,
                             safe,
                             RX_DATASET,
                             optional=False)
            else:
                validate_str("inputdata",
                             param,
                             safe,
                             RX_DATASET,
                             optional=True)
            validate_num("useparent", param, safe, optional=True)
            validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
            safe.kwargs['siteblacklist'] = self._expandSites(
                safe.kwargs['siteblacklist'])
            validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
            safe.kwargs['sitewhitelist'] = self._expandSites(
                safe.kwargs['sitewhitelist'])
            validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
            validate_num("algoargs", param, safe, optional=False)
            validate_num("totalunits", param, safe, optional=True)
            validate_str("cachefilename",
                         param,
                         safe,
                         RX_CACHENAME,
                         optional=False)
            validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
            validate_str(
                "lfnprefix", param, safe, RX_LFNPATH, optional=True
            )  #keeping this for compatibility with old clients. Will be converted into a full lfn
            validate_str("lfn", param, safe, RX_LFN, optional=True)
            self._checkOutLFN(safe.kwargs)
            validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
            validate_strlist("userfiles", param, safe, RX_USERFILE)
            validate_num("savelogsflag", param, safe, optional=False)
            validate_num("saveoutput", param, safe, optional=True)
            validate_num("faillimit", param, safe, optional=True)
            validate_num("ignorelocality", param, safe, optional=True)
            if safe.kwargs[
                    'ignorelocality'] and self.centralcfg.centralconfig.get(
                        'ign-locality-blacklist', []):
                safe.kwargs['siteblacklist'] += self._expandSites(
                    self.centralcfg.centralconfig['ign-locality-blacklist'])
            validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
            validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
            validate_str("publishname",
                         param,
                         safe,
                         RX_PUBLISH,
                         optional=False)
            validate_str("publishdbsurl",
                         param,
                         safe,
                         RX_DBSURL,
                         optional=True)
            validate_num("publication", param, safe, optional=False)
            #if publication is set as true both publishDataName and publishDbsUrl are needed
            if safe.kwargs["publication"] and not (
                    bool(safe.kwargs["publishname"])
                    and bool(safe.kwargs["publishdbsurl"])):
                raise InvalidParameter(
                    "You need to set both publishDataName and publishDbsUrl parameters if you need the automatic publication"
                )
            #if one and only one between publishDataName and publishDbsUrl is set raise an error (we need both or none of them)
            validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
            self._checkASODestination(safe.kwargs['asyncdest'])
            # We no longer use this attribute, but keep it around for older client compatibility
            validate_num("blacklistT1", param, safe, optional=True)
            validate_num("oneEventMode", param, safe, optional=True)
            validate_num("priority", param, safe, optional=True)
            validate_num("maxjobruntime", param, safe, optional=True)
            validate_num("numcores", param, safe, optional=True)
            validate_num("maxmemory", param, safe, optional=True)
            validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
            validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("runs", param, safe, RX_RUNS)
            validate_strlist("lumis", param, safe, RX_LUMIRANGE)
            #validate_str("scheduler", param, safe, RX_SCHEDULER)
            if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
                raise InvalidParameter(
                    "The number of runs and the number of lumis lists are different"
                )
            validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
            validate_str("asourl", param, safe, RX_ASOURL, optional=True)
            validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
            validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
            validate_str("scheddname",
                         param,
                         safe,
                         RX_SCHEDD_NAME,
                         optional=True)
            validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)

        elif method in ['POST']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
            validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
            safe.kwargs['siteblacklist'] = self._expandSites(
                safe.kwargs['siteblacklist'])
            validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
            safe.kwargs['sitewhitelist'] = self._expandSites(
                safe.kwargs['sitewhitelist'])
            validate_numlist('jobids', param, safe)
            validate_num("priority", param, safe, optional=True)
            validate_num("maxjobruntime", param, safe, optional=True)
            validate_num("numcores", param, safe, optional=True)
            validate_num("maxmemory", param, safe, optional=True)

        elif method in ['GET']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=True)
            validate_str('subresource',
                         param,
                         safe,
                         RX_SUBRESTAT,
                         optional=True)
            validate_str('username', param, safe, RX_USERNAME, optional=True)
            validate_str('timestamp', param, safe, RX_DATE,
                         optional=True)  ## inserted by eric

            ## Used to determine how much information to return to the client for status.
            ## also used by report to determine if it has to check job states
            validate_num("verbose", param, safe, optional=True)

            ## used by get log, get data
            validate_num('limit', param, safe, optional=True)
            validate_num('exitcode', param, safe, optional=True)
            validate_numlist('jobids', param, safe)

            ## used by errors and report (short format in report means we do not query DBS)
            validate_num('shortformat', param, safe, optional=True)

            ## validation parameters
            if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
                raise InvalidParameter("Invalid input parameters")
            if safe.kwargs['subresource'] in [
                    'data', 'logs'
            ] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
                raise InvalidParameter(
                    "You need to specify the number of jobs to retrieve or their ids."
                )

        elif method in ['DELETE']:
            validate_str("workflow", param, safe, RX_UNIQUEWF, optional=False)
            validate_num("force", param, safe, optional=True)
            validate_numlist('jobids', param, safe)
Ejemplo n.º 14
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid()

        if method in ['PUT']:
            #TODO check optional parameter
            #TODO check all the regexp
            validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
            validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
            validate_numlist("outfileruns", param, safe)
            if len(safe.kwargs["outfileruns"]) != len(
                    safe.kwargs["outfilelumis"]):
                raise InvalidParameter(
                    "The number of runs and the number of lumis lists are different"
                )
            validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
            validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
            validate_num("pandajobid", param, safe, optional=False)
            validate_num("outsize", param, safe, optional=False)
            validate_str("publishdataname",
                         param,
                         safe,
                         RX_PUBLISH,
                         optional=False)
            validate_str("appver", param, safe, RX_CMSSW, optional=False)
            validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
            validate_str("checksummd5",
                         param,
                         safe,
                         RX_CHECKSUM,
                         optional=False)
            validate_str("checksumcksum",
                         param,
                         safe,
                         RX_CHECKSUM,
                         optional=False)
            validate_str("checksumadler32",
                         param,
                         safe,
                         RX_CHECKSUM,
                         optional=False)
            validate_str("outlocation",
                         param,
                         safe,
                         RX_CMSSITE,
                         optional=False)
            validate_str("outtmplocation",
                         param,
                         safe,
                         RX_CMSSITE,
                         optional=False)
            validate_str("acquisitionera",
                         param,
                         safe,
                         RX_WORKFLOW,
                         optional=False)  #TODO Do we really need this?
            validate_str(
                "outdatasetname", param, safe, RX_OUTDSLFN,
                optional=False)  #TODO temporary, need to come up with a regex
            validate_str("outlfn", param, safe, RX_LFN, optional=False)
            validate_num("events", param, safe, optional=False)
        elif method in ['POST']:
            raise NotImplementedError
        elif method in ['GET']:
            validate_str("taskname", param, safe, RX_WORKFLOW, optional=False)
            validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
        elif method in ['DELETE']:
            raise NotImplementedError
Ejemplo n.º 15
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid()

        if method in ['PUT']:
            #TODO check optional parameter
            #TODO check all the regexp
            validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
            validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
            validate_numlist("outfileruns", param, safe)
            if len(safe.kwargs["outfileruns"]) != len(
                    safe.kwargs["outfilelumis"]):
                raise InvalidParameter(
                    "The number of runs and the number of lumis lists are different"
                )
            validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
            validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
            validate_num("pandajobid", param, safe, optional=False)
            validate_num("outsize", param, safe, optional=False)
            validate_str("publishdataname",
                         param,
                         safe,
                         RX_PUBLISH,
                         optional=False)
            validate_str("appver", param, safe, RX_CMSSW, optional=False)
            validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
            validate_str("checksummd5",
                         param,
                         safe,
                         RX_CHECKSUM,
                         optional=False)
            validate_str("checksumcksum",
                         param,
                         safe,
                         RX_CHECKSUM,
                         optional=False)
            validate_str("checksumadler32",
                         param,
                         safe,
                         RX_CHECKSUM,
                         optional=False)
            validate_str("outlocation",
                         param,
                         safe,
                         RX_CMSSITE,
                         optional=False)
            validate_str("outtmplocation",
                         param,
                         safe,
                         RX_CMSSITE,
                         optional=False)
            validate_str("acquisitionera",
                         param,
                         safe,
                         RX_TASKNAME,
                         optional=False)  #TODO Do we really need this?
            validate_str(
                "outdatasetname", param, safe, RX_OUTDSLFN,
                optional=False)  #TODO temporary, need to come up with a regex
            validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
            validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
            validate_num("events", param, safe, optional=False)
            validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
            validate_num("directstageout", param, safe, optional=True)
            safe.kwargs["directstageout"] = 'T' if safe.kwargs[
                "directstageout"] else 'F'  #'F' if not provided
        elif method in ['POST']:
            validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
            validate_str("outlfn", param, safe, RX_LFN, optional=False)
            validate_str("filestate",
                         param,
                         safe,
                         RX_FILESTATE,
                         optional=False)
        elif method in ['GET']:
            validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
            validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
        elif method in ['DELETE']:
            authz_operator()
            validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
            validate_str("hours", param, safe, RX_HOURS, optional=True)
            if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
                raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
                                         " will be deleted. Only one of the two parameters can be specified.")
Ejemplo n.º 16
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        authz_login_valid()

        if method in ['PUT']:
            #TODO check optional parameter
            #TODO check all the regexp
            validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
            validate_strlist("outfilelumis", param, safe, RX_LUMILIST)
            validate_numlist("outfileruns", param, safe)
            if len(safe.kwargs["outfileruns"]) != len(
                    safe.kwargs["outfilelumis"]):
                raise InvalidParameter(
                    "The number of runs and the number of lumis lists are different"
                )
            validate_strlist("inparentlfns", param, safe, RX_PARENTLFN)
            validate_str("globalTag", param, safe, RX_GLOBALTAG, optional=True)
            validate_str("jobid", param, safe, RX_JOBID, optional=True)
            #TODO: for backward compatibility. Get rid of the pandajobid once all jobs using it are done (one month after this gets released)
            #(The following five lines can be deleted)
            validate_num("pandajobid", param, safe, optional=True)
            if bool(safe.kwargs["jobid"]) == bool(safe.kwargs["pandajobid"]):
                raise InvalidParameter(
                    "Only one among jobid and pandajobid should be set")
            #Oracle/cx_oracle/python stack does not like None for numbers, even if they are nullable
            if safe.kwargs["pandajobid"] == None:
                safe.kwargs["pandajobid"] = 0
            validate_num("outsize", param, safe, optional=False)
            validate_str("publishdataname",
                         param,
                         safe,
                         RX_PUBLISH,
                         optional=False)
            validate_str("appver", param, safe, RX_CMSSW, optional=False)
            validate_str("outtype", param, safe, RX_OUTTYPES, optional=False)
            validate_str("checksummd5",
                         param,
                         safe,
                         RX_CHECKSUM,
                         optional=False)
            validate_str("checksumcksum",
                         param,
                         safe,
                         RX_CHECKSUM,
                         optional=False)
            validate_str("checksumadler32",
                         param,
                         safe,
                         RX_CHECKSUM,
                         optional=False)
            validate_str("outlocation",
                         param,
                         safe,
                         RX_CMSSITE,
                         optional=False)
            validate_str("outtmplocation",
                         param,
                         safe,
                         RX_CMSSITE,
                         optional=False)
            validate_str("acquisitionera",
                         param,
                         safe,
                         RX_TASKNAME,
                         optional=False)  #TODO Do we really need this?
            validate_str(
                "outdatasetname", param, safe, RX_OUTDSLFN,
                optional=False)  #TODO temporary, need to come up with a regex
            validate_str("outlfn", param, safe, RX_PARENTLFN, optional=False)
            validate_str("outtmplfn", param, safe, RX_PARENTLFN, optional=True)
            validate_num("events", param, safe, optional=False)
            validate_str("filestate", param, safe, RX_FILESTATE, optional=True)
            validate_num("directstageout", param, safe, optional=True)
            safe.kwargs["directstageout"] = 'T' if safe.kwargs[
                "directstageout"] else 'F'  #'F' if not provided
        elif method in ['POST']:
            validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
            validate_str("outlfn", param, safe, RX_LFN, optional=False)
            validate_str("filestate",
                         param,
                         safe,
                         RX_FILESTATE,
                         optional=False)
        elif method in ['GET']:
            validate_str("taskname", param, safe, RX_TASKNAME, optional=False)
            validate_str("filetype", param, safe, RX_OUTTYPES, optional=False)
            validate_num("howmany", param, safe, optional=True)
            validate_strlist("lfn", param, safe, RX_LFN)
        elif method in ['DELETE']:
            authz_operator()
            validate_str("taskname", param, safe, RX_TASKNAME, optional=True)
            validate_str("hours", param, safe, RX_HOURS, optional=True)
            if bool(safe.kwargs["taskname"]) == bool(safe.kwargs["hours"]):
                raise InvalidParameter("You have to specify a taskname or a number of hours. Files of this task or created before the number of hours"+\
                                         " will be deleted. Only one of the two parameters can be specified.")
Ejemplo n.º 17
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""

        if method in ['PUT']:
            username = cherrypy.request.user['login'] # username registered in SiteDB
            requestname = param.kwargs['workflow']
            param.kwargs['workflow'] = generateTaskName(username, requestname)
            validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
            validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
            validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
            # TODO this should be changed to be non-optional
            validate_str("generator", param, safe, RX_GENERATOR, optional=True)
            validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
            validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
            validate_num("nonprodsw", param, safe, optional=False)
            validate_str("jobarch", param, safe, RX_ARCH, optional=False)
            if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
                self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
            validate_num("useparent", param, safe, optional=True)
            validate_str("secondarydata", param, safe, RX_DATASET, optional=True)
            validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
            safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
            validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
            safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
            validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
            validate_num("algoargs", param, safe, optional=False)
            validate_num("totalunits", param, safe, optional=True)
            validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
            validate_str("debugfilename", param, safe, RX_CACHENAME, optional=True)
            validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
            validate_str("lfn", param, safe, RX_LFN, optional=True)
            self._checkOutLFN(safe.kwargs, username)
            validate_strlist("addoutputfiles", param, safe, RX_ADDFILE, custom_err="Incorrect 'JobType.outputFiles' parameter. " \
                    "Allowed regexp: '%s'." % RX_ADDFILE.pattern)
            validate_strlist("userfiles", param, safe, RX_USERFILE)
            validate_num("savelogsflag", param, safe, optional=False)
            validate_num("saveoutput", param, safe, optional=True)
            validate_num("faillimit", param, safe, optional=True)
            validate_num("ignorelocality", param, safe, optional=True)
            if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
                safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
            validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
            validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
            validate_num("publication", param, safe, optional=False)
            validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication'])))

            ## We might want to remove publishname once the backward compatibility
            ## wont be needed anymore. Then we can just keep publishname2
            ## NB: AFAIK the only client not using the CRABLibrary but direct cutl is HC,
            ## therefore we will need to make sure we do not break it!
            ## The following two lines will be removed in the future once we will
            ## not need backward compatibility anymore
            self._checkPublishDataName(param.kwargs, safe.kwargs['lfn'], requestname, username)
            validate_str('publishname', param, safe, RX_ANYTHING, optional=True)

            ##And this if as well, just do self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
            if not safe.kwargs["publishname"]: #new clients won't define this anymore
                ## The (user specified part of the) publication dataset name must be
                ## specified and must pass DBS validation. Since this is the correct
                ## validation function, it must be done before the
                ## validate_str("publishname", ...) we have below.
                self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
            else:
                param.kwargs["publishname2"] = safe.kwargs["publishname"]

            ## 'publishname' was already validated above in _checkPublishDataName().
            ## Calling validate_str with a fake regexp to move the param to the
            ## list of validated inputs
            validate_str("publishname2", param, safe, RX_ANYTHING, optional=True)

            validate_num("publishgroupname", param, safe, optional=True)

            if safe.kwargs['jobtype'] == 'PrivateMC':
                if param.kwargs['inputdata']:
                    msg  = "Invalid 'inputdata' parameter."
                    msg += " Job type PrivateMC does not take any input dataset."
                    msg += " If you really intend to run over an input dataset, then you must use job type Analysis."
                    raise InvalidParameter(msg)
                if safe.kwargs['userfiles']:
                    msg  = "Invalid 'userfiles' parameter."
                    msg += " Job type PrivateMC does not take any input files."
                    msg += " If you really intend to run over input files, then you must use job type Analysis."
                    raise InvalidParameter(msg)

            ## Client versions < 3.3.1511 may put in the input dataset something that is not
            ## really an input dataset (for PrivateMC or user input files). So the only case
            ## in which we are sure that we have to validate the input dataset is when the
            ## workflow type is Analysis, the workflow does not run on user input files and
            ## an input dataset is defined (scriptExe may not define an input).
            ## Once we don't care anymore about backward compatibility with client < 3.3.1511,
            ## we can uncomment the 1st line below and delete the next 4 lines.
            #validate_str("inputdata", param, safe, RX_DATASET, optional=True)
            if safe.kwargs['jobtype'] == 'Analysis' and not safe.kwargs['userfiles'] and 'inputdata' in param.kwargs:
                validate_str("inputdata", param, safe, RX_DATASET, optional=True)
            else:
                validate_str("inputdata", param, safe, RX_ANYTHING, optional=True)

            ## The client is not forced to define the primary dataset. So make sure to have
            ## defaults or take it from the input dataset. The primary dataset is needed for
            ## the LFN of the output/log files and for publication. We want to have it well
            ## defined even if publication and/or transfer to storage are off.
            if safe.kwargs['inputdata']:
                param.kwargs['primarydataset'] = safe.kwargs['inputdata'].split('/')[1]
            if not param.kwargs.get('primarydataset', None):
                if safe.kwargs['jobtype'] == 'PrivateMC':
                    param.kwargs['primarydataset'] = "CRAB_PrivateMC"
                elif safe.kwargs['jobtype'] == 'Analysis' and safe.kwargs['userfiles']:
                    param.kwargs['primarydataset'] = "CRAB_UserFiles"
                else:
                    param.kwargs['primarydataset'] = "CRAB_NoInput"
            ## We validate the primary dataset agains DBS rules even if publication is off,
            ## because in the future we may want to give the possibility to users to publish
            ## a posteriori.
            self._checkPrimaryDataset(param.kwargs, optional=False)
            validate_str("primarydataset", param, safe, RX_LFNPRIMDS, optional=False)

            validate_num("nonvaliddata", param, safe, optional=True)
            #if one and only one between outputDatasetTag and publishDbsUrl is set raise an error (we need both or none of them)
            validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
            self._checkASODestination(safe.kwargs['asyncdest'])
            # We no longer use this attribute, but keep it around for older client compatibility
            validate_num("blacklistT1", param, safe, optional=True)
            validate_num("oneEventMode", param, safe, optional=True)
            validate_num("priority", param, safe, optional=True)
            validate_num("maxjobruntime", param, safe, optional=True)
            validate_num("numcores", param, safe, optional=True)
            validate_num("maxmemory", param, safe, optional=True)
            validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
            validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect tfileoutfiles parameter (TFileService). " \
                    "Allowed regexp: '%s'." % RX_OUTFILES.pattern)
            validate_strlist("edmoutfiles", param, safe, RX_OUTFILES, custom_err="Incorrect edmoutfiles parameter (PoolOutputModule). " \
                    "Allowed regexp: '%s'." % RX_OUTFILES.pattern)
            validate_strlist("runs", param, safe, RX_RUNS)
            validate_strlist("lumis", param, safe, RX_LUMIRANGE)
            #validate_str("scheduler", param, safe, RX_SCHEDULER)
            if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
                raise InvalidParameter("The number of runs and the number of lumis lists are different")
            validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
            validate_str("asourl", param, safe, RX_ASOURL, optional=True)
            validate_str("asodb", param, safe, RX_ASODB, optional=True)
            safe.kwargs["asourl"], safe.kwargs["asodb"] = self._getAsoConfig(safe.kwargs["asourl"], safe.kwargs["asodb"])
            validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
            validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
            validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
            validate_str("collector", param, safe, RX_COLLECTOR, optional=True)
            validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
            validate_num("dryrun", param, safe, optional=True)
            validate_num("ignoreglobalblacklist", param, safe, optional=True)

        elif method in ['POST']:
            validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
            validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True)
            validate_numlist('jobids', param, safe)
            ## In a resubmission, the site black- and whitelists need to be interpreted
            ## differently than in an initial task submission. If there is no site black-
            ## or whitelist, set it to None and DataWorkflow will use the corresponding
            ## list defined in the initial task submission. If the site black- or whitelist
            ## is equal to the string 'empty', set it to an empty list and don't call
            ## validate_strlist as it would fail.
            if 'siteblacklist' not in param.kwargs:
                safe.kwargs['siteblacklist'] = None
            elif param.kwargs['siteblacklist'] == 'empty':
                safe.kwargs['siteblacklist'] = []
                del param.kwargs['siteblacklist']
            else:
                validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
                safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
            if 'sitewhitelist' not in param.kwargs:
                safe.kwargs['sitewhitelist'] = None
            elif param.kwargs['sitewhitelist'] == 'empty':
                safe.kwargs['sitewhitelist'] = []
                del param.kwargs['sitewhitelist']
            else:
                validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
                safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
            validate_num("maxjobruntime", param, safe, optional=True)
            validate_num("maxmemory", param, safe, optional=True)
            validate_num("numcores", param, safe, optional=True)
            validate_num("priority", param, safe, optional=True)
            validate_num("force", param, safe, optional=True)
            validate_num("publication", param, safe, optional=True)

        elif method in ['GET']:
            validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
            validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
            validate_str('username', param, safe, RX_USERNAME, optional=True)
            validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric

            ## Used to determine how much information to return to the client for status.
            ## also used by report to determine if it has to check job states
            validate_num("verbose", param, safe, optional=True)

            ## used by get log, get data
            validate_num('limit', param, safe, optional=True)
            validate_num('exitcode', param, safe, optional=True)
            validate_numlist('jobids', param, safe)

            ## used by errors and report (short format in report means we do not query DBS)
            validate_num('shortformat', param, safe, optional=True)

            ## validation parameters
            if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
                raise InvalidParameter("Invalid input parameters")
            if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
                raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")

        elif method in ['DELETE']:
            validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
            validate_num("force", param, safe, optional=True)
            validate_numlist('jobids', param, safe)
            validate_str("killwarning", param, safe,  RX_TEXT_FAIL, optional=True)
            #decode killwarning message if present
            if safe.kwargs['killwarning']:
                try:
                    safe.kwargs['killwarning'] = b64decode(safe.kwargs['killwarning'])
                except TypeError:
                    raise InvalidParameter("Failure message is not in the accepted format")
Ejemplo n.º 18
0
    def validate(self, apiobj, method, api, param, safe):
        """Validating all the input parameter as enforced by the WMCore.REST module"""
        #authz_login_valid()

        if method in ['PUT']:
            ## Define the taskname and write it in the 'workflow' parameter.
            timestamp = time.strftime('%y%m%d_%H%M%S', time.gmtime())
            username = cherrypy.request.user['login'] # username registered in SiteDB
            requestname = param.kwargs['workflow']
            taskname = "%s:%s_%s" % (timestamp, username, requestname)
            param.kwargs['workflow'] = taskname
            validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
            validate_str("activity", param, safe, RX_ACTIVITY, optional=True)
            validate_str("jobtype", param, safe, RX_JOBTYPE, optional=False)
            # TODO this should be changed to be non-optional
            validate_str("generator", param, safe, RX_GENERATOR, optional=True)
            validate_str("eventsperlumi", param, safe, RX_LUMIEVENTS, optional=True)
            validate_str("jobsw", param, safe, RX_CMSSW, optional=False)
            validate_num("nonprodsw", param, safe, optional=False)
            validate_str("jobarch", param, safe, RX_ARCH, optional=False)
            if not safe.kwargs["nonprodsw"]: #if the user wants to allow non-production releases
                self._checkReleases(safe.kwargs['jobarch'], safe.kwargs['jobsw'])
            validate_num("useparent", param, safe, optional=True)
            validate_str("secondarydata", param, safe, RX_DATASET, optional=True)
            validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
            safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
            validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
            safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
            validate_str("splitalgo", param, safe, RX_SPLIT, optional=False)
            validate_num("algoargs", param, safe, optional=False)
            validate_num("totalunits", param, safe, optional=True)
            validate_str("cachefilename", param, safe, RX_CACHENAME, optional=False)
            validate_str("cacheurl", param, safe, RX_CACHEURL, optional=False)
            validate_str("lfn", param, safe, RX_LFN, optional=True)
            self._checkOutLFN(safe.kwargs, username)
            validate_strlist("addoutputfiles", param, safe, RX_ADDFILE)
            validate_strlist("userfiles", param, safe, RX_USERFILE)
            validate_num("savelogsflag", param, safe, optional=False)
            validate_num("saveoutput", param, safe, optional=True)
            validate_num("faillimit", param, safe, optional=True)
            validate_num("ignorelocality", param, safe, optional=True)
            if safe.kwargs['ignorelocality'] and self.centralcfg.centralconfig.get('ign-locality-blacklist', []):
                safe.kwargs['siteblacklist'] += self._expandSites(self.centralcfg.centralconfig['ign-locality-blacklist'])
            validate_str("vorole", param, safe, RX_VOPARAMS, optional=True)
            validate_str("vogroup", param, safe, RX_VOPARAMS, optional=True)
            validate_num("publication", param, safe, optional=False)
            validate_str("publishdbsurl", param, safe, RX_DBSURL, optional=(not bool(safe.kwargs['publication'])))

            ## We might want to remove publishname once the backward compatibility
            ## wont be needed anymore. Then we can just keep publishname2
            ## NB: AFAIK the only client not using the CRABLibrary but direct cutl is HC,
            ## therefore we will need to make sure we do not break it!
            ## The following two lines will be removed in the future once we will
            ## not need backward compatibility anymore
            self._checkPublishDataName(param.kwargs, safe.kwargs['lfn'], requestname, username)
            validate_str('publishname', param, safe, RX_ANYTHING, optional=True)

            ##And this if as well, just do self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
            if not safe.kwargs["publishname"]: #new clients won't define this anymore
                ## The (user specified part of the) publication dataset name must be
                ## specified and must pass DBS validation. Since this is the correct
                ## validation function, it must be done before the
                ## validate_str("publishname", ...) we have below.
                self._checkPublishDataName2(param.kwargs, safe.kwargs['lfn'], requestname, username)
            else:
                param.kwargs["publishname2"] = safe.kwargs["publishname"]

            ## 'publishname' was already validated above in _checkPublishDataName().
            ## Calling validate_str with a fake regexp to move the param to the
            ## list of validated inputs
            validate_str("publishname2", param, safe, RX_ANYTHING, optional=True)

            validate_num("publishgroupname", param, safe, optional=True)

            if safe.kwargs['jobtype'] == 'PrivateMC':
                if param.kwargs['inputdata']:
                    msg  = "Invalid 'inputdata' parameter."
                    msg += " Job type PrivateMC does not take any input dataset."
                    msg += " If you really intend to run over an input dataset, then you must use job type Analysis."
                    raise InvalidParameter(msg)
                if safe.kwargs['userfiles']:
                    msg  = "Invalid 'userfiles' parameter."
                    msg += " Job type PrivateMC does not take any input files."
                    msg += " If you really intend to run over input files, then you must use job type Analysis."
                    raise InvalidParameter(msg)

            ## Client versions < 3.3.1511 may put in the input dataset something that is not
            ## really an input dataset (for PrivateMC or user input files). So the only case
            ## in which we are sure that we have to validate the input dataset is when the
            ## workflow type is Analysis, the workflow does not run on user input files and
            ## an input dataset is defined (scriptExe may not define an input).
            ## Once we don't care anymore about backward compatibility with client < 3.3.1511,
            ## we can uncomment the 1st line below and delete the next 4 lines.
            #validate_str("inputdata", param, safe, RX_DATASET, optional=True)
            if safe.kwargs['jobtype'] == 'Analysis' and not safe.kwargs['userfiles'] and 'inputdata' in param.kwargs:
                validate_str("inputdata", param, safe, RX_DATASET, optional=True)
            else:
                validate_str("inputdata", param, safe, RX_ANYTHING, optional=True)

            ## The client is not forced to define the primary dataset. So make sure to have
            ## defaults or take it from the input dataset. The primary dataset is needed for
            ## the LFN of the output/log files and for publication. We want to have it well
            ## defined even if publication and/or transfer to storage are off. 
            if safe.kwargs['inputdata']:
                param.kwargs['primarydataset'] = safe.kwargs['inputdata'].split('/')[1]
            if not param.kwargs.get('primarydataset', None):
                if safe.kwargs['jobtype'] == 'PrivateMC':
                    param.kwargs['primarydataset'] = "CRAB_PrivateMC"
                elif safe.kwargs['jobtype'] == 'Analysis' and safe.kwargs['userfiles']:
                    param.kwargs['primarydataset'] = "CRAB_UserFiles"
                else:
                    param.kwargs['primarydataset'] = "CRAB_NoInput"
            ## We validate the primary dataset agains DBS rules even if publication is off,
            ## because in the future we may want to give the possibility to users to publish
            ## a posteriori.
            self._checkPrimaryDataset(param.kwargs, optional=False)
            validate_str("primarydataset", param, safe, RX_LFNPRIMDS, optional=False)

            validate_num("nonvaliddata", param, safe, optional=True)
            #if one and only one between outputDatasetTag and publishDbsUrl is set raise an error (we need both or none of them)
            validate_str("asyncdest", param, safe, RX_CMSSITE, optional=False)
            self._checkASODestination(safe.kwargs['asyncdest'])
            # We no longer use this attribute, but keep it around for older client compatibility
            validate_num("blacklistT1", param, safe, optional=True)
            validate_num("oneEventMode", param, safe, optional=True)
            validate_num("priority", param, safe, optional=True)
            validate_num("maxjobruntime", param, safe, optional=True)
            validate_num("numcores", param, safe, optional=True)
            validate_num("maxmemory", param, safe, optional=True)
            validate_str("dbsurl", param, safe, RX_DBSURL, optional=False)
            validate_strlist("tfileoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("edmoutfiles", param, safe, RX_OUTFILES)
            validate_strlist("runs", param, safe, RX_RUNS)
            validate_strlist("lumis", param, safe, RX_LUMIRANGE)
            #validate_str("scheduler", param, safe, RX_SCHEDULER)
            if len(safe.kwargs["runs"]) != len(safe.kwargs["lumis"]):
                raise InvalidParameter("The number of runs and the number of lumis lists are different")
            validate_strlist("adduserfiles", param, safe, RX_ADDFILE)
            validate_str("asourl", param, safe, RX_ASOURL, optional=True)
            validate_str("scriptexe", param, safe, RX_ADDFILE, optional=True)
            validate_strlist("scriptargs", param, safe, RX_SCRIPTARGS)
            validate_str("scheddname", param, safe, RX_SCHEDD_NAME, optional=True)
            validate_str("collector", param, safe, RX_COLLECTOR, optional=True)
            validate_strlist("extrajdl", param, safe, RX_SCRIPTARGS)
            validate_num("dryrun", param, safe, optional=True)

        elif method in ['POST']:
            validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
            validate_str("subresource", param, safe, RX_SUBRESTAT, optional=True)
            validate_numlist('jobids', param, safe)
            ## In a resubmission, the site black- and whitelists need to be interpreted
            ## differently than in an initial task submission. If there is no site black-
            ## or whitelist, set it to None and DataWorkflow will use the corresponding
            ## list defined in the initial task submission. If the site black- or whitelist
            ## is equal to the string 'empty', set it to an empty list and don't call
            ## validate_strlist as it would fail.
            if 'siteblacklist' not in param.kwargs:
                safe.kwargs['siteblacklist'] = None
            elif param.kwargs['siteblacklist'] == 'empty':
                safe.kwargs['siteblacklist'] = []
                del param.kwargs['siteblacklist']
            else:
                validate_strlist("siteblacklist", param, safe, RX_CMSSITE)
                safe.kwargs['siteblacklist'] = self._expandSites(safe.kwargs['siteblacklist'])
            if 'sitewhitelist' not in param.kwargs:
                safe.kwargs['sitewhitelist'] = None
            elif param.kwargs['sitewhitelist'] == 'empty':
                safe.kwargs['sitewhitelist'] = []
                del param.kwargs['sitewhitelist']
            else:
                validate_strlist("sitewhitelist", param, safe, RX_CMSSITE)
                safe.kwargs['sitewhitelist'] = self._expandSites(safe.kwargs['sitewhitelist'])
            validate_num("maxjobruntime", param, safe, optional=True)
            validate_num("maxmemory", param, safe, optional=True)
            validate_num("numcores", param, safe, optional=True)
            validate_num("priority", param, safe, optional=True)
            validate_num("force", param, safe, optional=True)
            validate_num("publication", param, safe, optional=True)

        elif method in ['GET']:
            validate_str("workflow", param, safe, RX_TASKNAME, optional=True)
            validate_str('subresource', param, safe, RX_SUBRESTAT, optional=True)
            validate_str('username', param, safe, RX_USERNAME, optional=True)
            validate_str('timestamp', param, safe, RX_DATE, optional=True) ## inserted by eric

            ## Used to determine how much information to return to the client for status.
            ## also used by report to determine if it has to check job states
            validate_num("verbose", param, safe, optional=True)

            ## used by get log, get data
            validate_num('limit', param, safe, optional=True)
            validate_num('exitcode', param, safe, optional=True)
            validate_numlist('jobids', param, safe)

            ## used by errors and report (short format in report means we do not query DBS)
            validate_num('shortformat', param, safe, optional=True)

            ## validation parameters
            if not safe.kwargs['workflow'] and safe.kwargs['subresource']:
                raise InvalidParameter("Invalid input parameters")
            if safe.kwargs['subresource'] in ['data', 'logs'] and not safe.kwargs['limit'] and not safe.kwargs['jobids']:
                raise InvalidParameter("You need to specify the number of jobs to retrieve or their ids.")

        elif method in ['DELETE']:
            validate_str("workflow", param, safe, RX_TASKNAME, optional=False)
            validate_num("force", param, safe, optional=True)
            validate_numlist('jobids', param, safe)