def validateSchema(self, schema): """ Since we skip the master validation for Resubmission specs, we better have some specific validation """ if schema.get("ResubmissionCount", 1) > 1: # we cannot validate such schema return # load assignment + creation + resubmission creation args definition argumentDefinition = self.getWorkloadAssignArgs() parentSpecClass = loadSpecClassByType(schema['OriginalRequestType']) argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs()) argumentDefinition.update(self.getWorkloadCreateArgs()) try: validateArgumentsCreate(schema, argumentDefinition) except Exception as ex: self.raiseValidationException(str(ex)) # and some extra validation based on the parent workflow if schema['OriginalRequestType'] != "TaskChain": for param in ("TimePerEvent", "Memory"): if isinstance(schema.get(param), dict): msg = "ACDC for parent spec of type: {} ".format( schema['OriginalRequestType']) msg += "cannot have parameter: {} defined as a dictionary: {}".format( param, schema[param]) self.raiseValidationException(msg)
def validateSchema(self, schema): """ _validateSchema_ Check for required fields, and some skim facts """ DataProcessing.validateSchema(self, schema) mainOutputModules = list( self.validateConfigCacheExists( configID=schema["ConfigCacheID"], configCacheUrl=schema['ConfigCacheUrl'], couchDBName=schema["CouchDBName"], getOutputModules=True)) # Skim facts have to be validated outside the usual master validation skimSchema = { k: v for (k, v) in viewitems(schema) if k.startswith("Skim") } skimArguments = self.getSkimArguments() skimIndex = 1 skimInputs = set() while "SkimName%s" % skimIndex in schema: instanceArguments = {} for argument in skimArguments: realArg = argument.replace("#N", str(skimIndex)) instanceArguments[realArg] = skimArguments[argument] try: validateArgumentsCreate(skimSchema, instanceArguments) except Exception as ex: self.raiseValidationException(str(ex)) self.validateConfigCacheExists( configID=schema["Skim%sConfigCacheID" % skimIndex], configCacheUrl=schema['ConfigCacheUrl'], couchDBName=schema["CouchDBName"], getOutputModules=False) if schema["SkimInput%s" % skimIndex] not in mainOutputModules: error = "Processing config does not have the following output module: %s." % schema[ "SkimInput%s" % skimIndex] self.raiseValidationException(msg=error) skimInputs.add(schema["SkimInput%s" % skimIndex]) skimIndex += 1 # Validate that the transient output modules are used in a skim task if "TransientOutputModules" in schema: diffSet = set(schema["TransientOutputModules"]) - skimInputs if diffSet: self.raiseValidationException( msg= "A transient output module was specified but no skim was defined for it" )
def validateStep(self, taskConf, taskArgumentDefinition): """ _validateStep_ Validate the step information against the given argument description """ try: validateArgumentsCreate(taskConf, taskArgumentDefinition, checkInputDset=False) except Exception as ex: self.raiseValidationException(str(ex)) return
def validateTask(self, taskConf, taskArgumentDefinition): """ _validateTask_ Validate the task information against the given argument description """ try: validateArgumentsCreate(taskConf, taskArgumentDefinition, checkInputDset=False) except Exception as ex: self.raiseValidationException(str(ex)) return
def validateStep(self, taskConf, taskArgumentDefinition): """ _validateStep_ Validate the step information against the given argument description """ try: validateArgumentsCreate(taskConf, taskArgumentDefinition, checkInputDset=False) except WMSpecFactoryException: # just re-raise it to keep the error message clear raise except Exception as ex: self.raiseValidationException(str(ex)) return
def validateTask(self, taskConf, taskArgumentDefinition): """ _validateTask_ Validate the task information against the given argument description """ try: validateArgumentsCreate(taskConf, taskArgumentDefinition, checkInputDset=False) # Validate GPU-related spec parameters StdBase.validateGPUSettings(taskConf) except WMSpecFactoryException: # just re-raise it to keep the error message clear raise except Exception as ex: self.raiseValidationException(str(ex))
def validateSchema(self, schema): """ Since we skip the master validation for Resubmission specs, we better have some specific validation """ if schema.get('OriginalRequestType') == 'Resubmission': # we cannot validate such schema return # load assignment + creation + resubmission creation args definition argumentDefinition = self.getWorkloadAssignArgs() parentSpecClass = loadSpecClassByType(schema['OriginalRequestType']) argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs()) argumentDefinition.update(self.getWorkloadCreateArgs()) try: validateArgumentsCreate(schema, argumentDefinition) except Exception as ex: self.raiseValidationException(str(ex))
def validateSchema(self, schema): """ _validateSchema_ Check for required fields, and some skim facts """ DataProcessing.validateSchema(self, schema) mainOutputModules = self.validateConfigCacheExists(configID = schema["ConfigCacheID"], configCacheUrl = schema['ConfigCacheUrl'], couchDBName = schema["CouchDBName"], getOutputModules = True).keys() # Skim facts have to be validated outside the usual master validation skimSchema = {k: v for (k, v) in schema.iteritems() if k.startswith("Skim")} skimArguments = self.getSkimArguments() skimIndex = 1 skimInputs = set() while "SkimName%s" % skimIndex in schema: instanceArguments = {} for argument in skimArguments: realArg = argument.replace("#N", str(skimIndex)) instanceArguments[realArg] = skimArguments[argument] try: validateArgumentsCreate(skimSchema, instanceArguments) except Exception as ex: self.raiseValidationException(str(ex)) self.validateConfigCacheExists(configID = schema["Skim%sConfigCacheID" % skimIndex], configCacheUrl = schema['ConfigCacheUrl'], couchDBName = schema["CouchDBName"], getOutputModules=False) if schema["SkimInput%s" % skimIndex] not in mainOutputModules: error = "Processing config does not have the following output module: %s." % schema["SkimInput%s" % skimIndex] self.raiseValidationException(msg = error) skimInputs.add(schema["SkimInput%s" % skimIndex]) skimIndex += 1 # Validate that the transient output modules are used in a skim task if "TransientOutputModules" in schema: diffSet = set(schema["TransientOutputModules"]) - skimInputs if diffSet: self.raiseValidationException(msg = "A transient output module was specified but no skim was defined for it")
def validateTask(self, taskConf, taskArgumentDefinition): """ _validateTask_ Validate the task information against the given argument description """ msg = validateArgumentsCreate(taskConf, taskArgumentDefinition) if msg is not None: self.raiseValidationException(msg) # Also retrieve the "main" arguments which may be overriden in the task # Change them all to optional for validation baseArgs = self.getWorkloadArguments() validateArgumentsNoOptionalCheck(taskConf, baseArgs) for arg in baseArgs: baseArgs[arg]["optional"] = True msg = validateArgumentsCreate(taskConf, baseArgs) if msg is not None: self.raiseValidationException(msg) return
def validateSchema(self, schema): """ Since we skip the master validation for Resubmission specs, we better have some specific validation """ #TODO: for the legacy code if ACDC is created before the updated. remove in next release (Mar 2018) if schema.get('OriginalRequestType') == 'Resubmission': # we cannot validate such schema return if schema.get("ResubmissionCount", 1) > 1: # we cannot validate such schema return # load assignment + creation + resubmission creation args definition argumentDefinition = self.getWorkloadAssignArgs() parentSpecClass = loadSpecClassByType(schema['OriginalRequestType']) argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs()) argumentDefinition.update(self.getWorkloadCreateArgs()) try: validateArgumentsCreate(schema, argumentDefinition) except Exception as ex: self.raiseValidationException(str(ex))
def validateSchema(self, schema): """ _validateSchema_ Check for required fields, and some skim facts """ DataProcessing.validateSchema(self, schema) couchUrl = schema.get("ConfigCacheUrl", None) or schema["CouchURL"] mainOutputModules = self.validateConfigCacheExists( configID=schema["ConfigCacheID"], couchURL=couchUrl, couchDBName=schema["CouchDBName"], getOutputModules=True).keys() # Skim facts have to be validated outside the usual master validation skimArguments = self.getSkimArguments() skimIndex = 1 skimInputs = set() while "SkimName%s" % skimIndex in schema: instanceArguments = {} for argument in skimArguments.keys(): realArg = argument.replace("#N", str(skimIndex)) instanceArguments[realArg] = skimArguments[argument] msg = validateArgumentsCreate(schema, instanceArguments) if msg is not None: self.raiseValidationException(msg) self.validateConfigCacheExists( configID=schema["Skim%sConfigCacheID" % skimIndex], couchURL=couchUrl, couchDBName=schema["CouchDBName"]) if schema["SkimInput%s" % skimIndex] not in mainOutputModules: error = "Processing config does not have the following output module: %s." % schema[ "SkimInput%s" % skimIndex] self.raiseValidationException(msg=error) skimInputs.add(schema["SkimInput%s" % skimIndex]) skimIndex += 1 # Validate that the transient output modules are used in a skim task if "TransientOutputModules" in schema: diffSet = set(schema["TransientOutputModules"]) - skimInputs if diffSet: self.raiseValidationException( msg= "A transient output module was specified but no skim was defined for it" )
def masterValidation(self, schema): """ _masterValidation_ This is validation for global inputs that have to be implemented for multiple types of workflows in the exact same way. This uses programatically the definitions in getWorkloadArguments for type-checking, existence, null tests and the specific validation functions. Any spec-specific extras are implemented in the overriden validateSchema """ # Validate the arguments according to the workload arguments definition argumentDefinition = self.getWorkloadArguments() msg = validateArgumentsCreate(schema, argumentDefinition) if msg is not None: self.raiseValidationException(msg) return