예제 #1
0
    def validateSchema(self, schema):
        """
        Since we skip the master validation for Resubmission specs, we better have
        some specific validation
        """
        if schema.get("ResubmissionCount", 1) > 1:
            # we cannot validate such schema
            return
        # load assignment + creation + resubmission creation args definition
        argumentDefinition = self.getWorkloadAssignArgs()
        parentSpecClass = loadSpecClassByType(schema['OriginalRequestType'])
        argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs())
        argumentDefinition.update(self.getWorkloadCreateArgs())

        try:
            validateArgumentsCreate(schema, argumentDefinition)
        except Exception as ex:
            self.raiseValidationException(str(ex))

        # and some extra validation based on the parent workflow
        if schema['OriginalRequestType'] != "TaskChain":
            for param in ("TimePerEvent", "Memory"):
                if isinstance(schema.get(param), dict):
                    msg = "ACDC for parent spec of type: {} ".format(
                        schema['OriginalRequestType'])
                    msg += "cannot have parameter: {} defined as a dictionary: {}".format(
                        param, schema[param])
                    self.raiseValidationException(msg)
예제 #2
0
    def validateSchema(self, schema):
        """
        _validateSchema_

        Check for required fields, and some skim facts
        """
        DataProcessing.validateSchema(self, schema)
        mainOutputModules = list(
            self.validateConfigCacheExists(
                configID=schema["ConfigCacheID"],
                configCacheUrl=schema['ConfigCacheUrl'],
                couchDBName=schema["CouchDBName"],
                getOutputModules=True))

        # Skim facts have to be validated outside the usual master validation
        skimSchema = {
            k: v
            for (k, v) in viewitems(schema) if k.startswith("Skim")
        }
        skimArguments = self.getSkimArguments()
        skimIndex = 1
        skimInputs = set()
        while "SkimName%s" % skimIndex in schema:
            instanceArguments = {}
            for argument in skimArguments:
                realArg = argument.replace("#N", str(skimIndex))
                instanceArguments[realArg] = skimArguments[argument]
            try:
                validateArgumentsCreate(skimSchema, instanceArguments)
            except Exception as ex:
                self.raiseValidationException(str(ex))

            self.validateConfigCacheExists(
                configID=schema["Skim%sConfigCacheID" % skimIndex],
                configCacheUrl=schema['ConfigCacheUrl'],
                couchDBName=schema["CouchDBName"],
                getOutputModules=False)
            if schema["SkimInput%s" % skimIndex] not in mainOutputModules:
                error = "Processing config does not have the following output module: %s." % schema[
                    "SkimInput%s" % skimIndex]
                self.raiseValidationException(msg=error)
            skimInputs.add(schema["SkimInput%s" % skimIndex])
            skimIndex += 1

        # Validate that the transient output modules are used in a skim task
        if "TransientOutputModules" in schema:
            diffSet = set(schema["TransientOutputModules"]) - skimInputs
            if diffSet:
                self.raiseValidationException(
                    msg=
                    "A transient output module was specified but no skim was defined for it"
                )
예제 #3
0
    def validateStep(self, taskConf, taskArgumentDefinition):
        """
        _validateStep_

        Validate the step information against the given
        argument description
        """
        try:
            validateArgumentsCreate(taskConf, taskArgumentDefinition, checkInputDset=False)
        except Exception as ex:
            self.raiseValidationException(str(ex))

        return
예제 #4
0
    def validateTask(self, taskConf, taskArgumentDefinition):
        """
        _validateTask_

        Validate the task information against the given
        argument description
        """
        try:
            validateArgumentsCreate(taskConf, taskArgumentDefinition, checkInputDset=False)
        except Exception as ex:
            self.raiseValidationException(str(ex))

        return
예제 #5
0
    def validateStep(self, taskConf, taskArgumentDefinition):
        """
        _validateStep_

        Validate the step information against the given
        argument description
        """
        try:
            validateArgumentsCreate(taskConf, taskArgumentDefinition, checkInputDset=False)
        except WMSpecFactoryException:
            # just re-raise it to keep the error message clear
            raise
        except Exception as ex:
            self.raiseValidationException(str(ex))

        return
예제 #6
0
    def validateTask(self, taskConf, taskArgumentDefinition):
        """
        _validateTask_

        Validate the task information against the given
        argument description
        """
        try:
            validateArgumentsCreate(taskConf,
                                    taskArgumentDefinition,
                                    checkInputDset=False)
            # Validate GPU-related spec parameters
            StdBase.validateGPUSettings(taskConf)
        except WMSpecFactoryException:
            # just re-raise it to keep the error message clear
            raise
        except Exception as ex:
            self.raiseValidationException(str(ex))
예제 #7
0
    def validateSchema(self, schema):
        """
        Since we skip the master validation for Resubmission specs, we better have
        some specific validation
        """
        if schema.get('OriginalRequestType') == 'Resubmission':
            # we cannot validate such schema
            return

        # load assignment + creation + resubmission creation args definition
        argumentDefinition = self.getWorkloadAssignArgs()
        parentSpecClass = loadSpecClassByType(schema['OriginalRequestType'])
        argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs())
        argumentDefinition.update(self.getWorkloadCreateArgs())

        try:
            validateArgumentsCreate(schema, argumentDefinition)
        except Exception as ex:
            self.raiseValidationException(str(ex))
예제 #8
0
    def validateSchema(self, schema):
        """
        Since we skip the master validation for Resubmission specs, we better have
        some specific validation
        """
        if schema.get('OriginalRequestType') == 'Resubmission':
            # we cannot validate such schema
            return

        # load assignment + creation + resubmission creation args definition
        argumentDefinition = self.getWorkloadAssignArgs()
        parentSpecClass = loadSpecClassByType(schema['OriginalRequestType'])
        argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs())
        argumentDefinition.update(self.getWorkloadCreateArgs())

        try:
            validateArgumentsCreate(schema, argumentDefinition)
        except Exception as ex:
            self.raiseValidationException(str(ex))
예제 #9
0
파일: ReReco.py 프로젝트: dmwm/WMCore
    def validateSchema(self, schema):
        """
        _validateSchema_

        Check for required fields, and some skim facts
        """
        DataProcessing.validateSchema(self, schema)
        mainOutputModules = self.validateConfigCacheExists(configID = schema["ConfigCacheID"],
                                                           configCacheUrl = schema['ConfigCacheUrl'],
                                                           couchDBName = schema["CouchDBName"],
                                                           getOutputModules = True).keys()

        # Skim facts have to be validated outside the usual master validation
        skimSchema = {k: v for (k, v) in schema.iteritems() if k.startswith("Skim")}
        skimArguments = self.getSkimArguments()
        skimIndex = 1
        skimInputs = set()
        while "SkimName%s" % skimIndex in schema:
            instanceArguments = {}
            for argument in skimArguments:
                realArg = argument.replace("#N", str(skimIndex))
                instanceArguments[realArg] = skimArguments[argument]
            try:
                validateArgumentsCreate(skimSchema, instanceArguments)
            except Exception as ex:
                self.raiseValidationException(str(ex))

            self.validateConfigCacheExists(configID = schema["Skim%sConfigCacheID" % skimIndex],
                                           configCacheUrl = schema['ConfigCacheUrl'],
                                           couchDBName = schema["CouchDBName"],
                                           getOutputModules=False)
            if schema["SkimInput%s" % skimIndex] not in mainOutputModules:
                error = "Processing config does not have the following output module: %s." % schema["SkimInput%s" % skimIndex]
                self.raiseValidationException(msg = error)
            skimInputs.add(schema["SkimInput%s" % skimIndex])
            skimIndex += 1

        # Validate that the transient output modules are used in a skim task
        if "TransientOutputModules" in schema:
            diffSet = set(schema["TransientOutputModules"]) - skimInputs
            if diffSet:
                self.raiseValidationException(msg = "A transient output module was specified but no skim was defined for it")
예제 #10
0
    def validateTask(self, taskConf, taskArgumentDefinition):
        """
        _validateTask_

        Validate the task information against the given
        argument description
        """
        msg = validateArgumentsCreate(taskConf, taskArgumentDefinition)
        if msg is not None:
            self.raiseValidationException(msg)

        # Also retrieve the "main" arguments which may be overriden in the task
        # Change them all to optional for validation
        baseArgs = self.getWorkloadArguments()
        validateArgumentsNoOptionalCheck(taskConf, baseArgs)

        for arg in baseArgs:
            baseArgs[arg]["optional"] = True
        msg = validateArgumentsCreate(taskConf, baseArgs)
        if msg is not None:
            self.raiseValidationException(msg)
        return
예제 #11
0
    def validateTask(self, taskConf, taskArgumentDefinition):
        """
        _validateTask_

        Validate the task information against the given
        argument description
        """
        msg = validateArgumentsCreate(taskConf, taskArgumentDefinition)
        if msg is not None:
            self.raiseValidationException(msg)

        # Also retrieve the "main" arguments which may be overriden in the task
        # Change them all to optional for validation
        baseArgs = self.getWorkloadArguments()
        validateArgumentsNoOptionalCheck(taskConf, baseArgs)

        for arg in baseArgs:
            baseArgs[arg]["optional"] = True
        msg = validateArgumentsCreate(taskConf, baseArgs)
        if msg is not None:
            self.raiseValidationException(msg)
        return
예제 #12
0
파일: Resubmission.py 프로젝트: dmwm/WMCore
    def validateSchema(self, schema):
        """
        Since we skip the master validation for Resubmission specs, we better have
        some specific validation
        """
        #TODO: for the legacy code if ACDC is created before the updated. remove in next release (Mar 2018)
        if schema.get('OriginalRequestType') == 'Resubmission':
            # we cannot validate such schema
            return

        if schema.get("ResubmissionCount", 1) > 1:
            # we cannot validate such schema
            return

        # load assignment + creation + resubmission creation args definition
        argumentDefinition = self.getWorkloadAssignArgs()
        parentSpecClass = loadSpecClassByType(schema['OriginalRequestType'])
        argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs())
        argumentDefinition.update(self.getWorkloadCreateArgs())

        try:
            validateArgumentsCreate(schema, argumentDefinition)
        except Exception as ex:
            self.raiseValidationException(str(ex))
예제 #13
0
    def validateSchema(self, schema):
        """
        Since we skip the master validation for Resubmission specs, we better have
        some specific validation
        """
        #TODO: for the legacy code if ACDC is created before the updated. remove in next release (Mar 2018)
        if schema.get('OriginalRequestType') == 'Resubmission':
            # we cannot validate such schema
            return

        if schema.get("ResubmissionCount", 1) > 1:
            # we cannot validate such schema
            return

        # load assignment + creation + resubmission creation args definition
        argumentDefinition = self.getWorkloadAssignArgs()
        parentSpecClass = loadSpecClassByType(schema['OriginalRequestType'])
        argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs())
        argumentDefinition.update(self.getWorkloadCreateArgs())

        try:
            validateArgumentsCreate(schema, argumentDefinition)
        except Exception as ex:
            self.raiseValidationException(str(ex))
예제 #14
0
파일: ReReco.py 프로젝트: tsarangi/WMCore
    def validateSchema(self, schema):
        """
        _validateSchema_

        Check for required fields, and some skim facts
        """
        DataProcessing.validateSchema(self, schema)
        couchUrl = schema.get("ConfigCacheUrl", None) or schema["CouchURL"]
        mainOutputModules = self.validateConfigCacheExists(
            configID=schema["ConfigCacheID"],
            couchURL=couchUrl,
            couchDBName=schema["CouchDBName"],
            getOutputModules=True).keys()

        # Skim facts have to be validated outside the usual master validation
        skimArguments = self.getSkimArguments()
        skimIndex = 1
        skimInputs = set()
        while "SkimName%s" % skimIndex in schema:
            instanceArguments = {}
            for argument in skimArguments.keys():
                realArg = argument.replace("#N", str(skimIndex))
                instanceArguments[realArg] = skimArguments[argument]
            msg = validateArgumentsCreate(schema, instanceArguments)
            if msg is not None:
                self.raiseValidationException(msg)

            self.validateConfigCacheExists(
                configID=schema["Skim%sConfigCacheID" % skimIndex],
                couchURL=couchUrl,
                couchDBName=schema["CouchDBName"])
            if schema["SkimInput%s" % skimIndex] not in mainOutputModules:
                error = "Processing config does not have the following output module: %s." % schema[
                    "SkimInput%s" % skimIndex]
                self.raiseValidationException(msg=error)
            skimInputs.add(schema["SkimInput%s" % skimIndex])
            skimIndex += 1

        # Validate that the transient output modules are used in a skim task
        if "TransientOutputModules" in schema:
            diffSet = set(schema["TransientOutputModules"]) - skimInputs
            if diffSet:
                self.raiseValidationException(
                    msg=
                    "A transient output module was specified but no skim was defined for it"
                )
예제 #15
0
파일: StdBase.py 프로젝트: lucacopa/WMCore
    def masterValidation(self, schema):
        """
        _masterValidation_

        This is validation for global inputs that have to be implemented for
        multiple types of workflows in the exact same way.

        This uses programatically the definitions in getWorkloadArguments
        for type-checking, existence, null tests and the specific validation functions.

        Any spec-specific extras are implemented in the overriden validateSchema
        """
        # Validate the arguments according to the workload arguments definition
        argumentDefinition = self.getWorkloadArguments()
        msg = validateArgumentsCreate(schema, argumentDefinition)
        if msg is not None:
            self.raiseValidationException(msg)
        return