def validateSchema(self, schema):
"""
Since we skip the master validation for Resubmission specs, we better have
some specific validation
"""
if schema.get("ResubmissionCount", 1) > 1:
# we cannot validate such schema
return
# load assignment + creation + resubmission creation args definition
argumentDefinition = self.getWorkloadAssignArgs()
parentSpecClass = loadSpecClassByType(schema['OriginalRequestType'])
argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs())
argumentDefinition.update(self.getWorkloadCreateArgs())
try:
validateArgumentsCreate(schema, argumentDefinition)
except Exception as ex:
self.raiseValidationException(str(ex))
# and some extra validation based on the parent workflow
if schema['OriginalRequestType'] != "TaskChain":
for param in ("TimePerEvent", "Memory"):
if isinstance(schema.get(param), dict):
msg = "ACDC for parent spec of type: {} ".format(
schema['OriginalRequestType'])
msg += "cannot have parameter: {} defined as a dictionary: {}".format(
param, schema[param])
self.raiseValidationException(msg)
def validateSchema(self, schema):
"""
_validateSchema_
Check for required fields, and some skim facts
"""
DataProcessing.validateSchema(self, schema)
mainOutputModules = list(
self.validateConfigCacheExists(
configID=schema["ConfigCacheID"],
configCacheUrl=schema['ConfigCacheUrl'],
couchDBName=schema["CouchDBName"],
getOutputModules=True))
# Skim facts have to be validated outside the usual master validation
skimSchema = {
k: v
for (k, v) in viewitems(schema) if k.startswith("Skim")
}
skimArguments = self.getSkimArguments()
skimIndex = 1
skimInputs = set()
while "SkimName%s" % skimIndex in schema:
instanceArguments = {}
for argument in skimArguments:
realArg = argument.replace("#N", str(skimIndex))
instanceArguments[realArg] = skimArguments[argument]
try:
validateArgumentsCreate(skimSchema, instanceArguments)
except Exception as ex:
self.raiseValidationException(str(ex))
self.validateConfigCacheExists(
configID=schema["Skim%sConfigCacheID" % skimIndex],
configCacheUrl=schema['ConfigCacheUrl'],
couchDBName=schema["CouchDBName"],
getOutputModules=False)
if schema["SkimInput%s" % skimIndex] not in mainOutputModules:
error = "Processing config does not have the following output module: %s." % schema[
"SkimInput%s" % skimIndex]
self.raiseValidationException(msg=error)
skimInputs.add(schema["SkimInput%s" % skimIndex])
skimIndex += 1
# Validate that the transient output modules are used in a skim task
if "TransientOutputModules" in schema:
diffSet = set(schema["TransientOutputModules"]) - skimInputs
if diffSet:
self.raiseValidationException(
msg=
"A transient output module was specified but no skim was defined for it"
)
def validateStep(self, taskConf, taskArgumentDefinition):
"""
_validateStep_
Validate the step information against the given
argument description
"""
try:
validateArgumentsCreate(taskConf, taskArgumentDefinition, checkInputDset=False)
except Exception as ex:
self.raiseValidationException(str(ex))
return
def validateTask(self, taskConf, taskArgumentDefinition):
"""
_validateTask_
Validate the task information against the given
argument description
"""
try:
validateArgumentsCreate(taskConf, taskArgumentDefinition, checkInputDset=False)
except Exception as ex:
self.raiseValidationException(str(ex))
return
def validateStep(self, taskConf, taskArgumentDefinition):
"""
_validateStep_
Validate the step information against the given
argument description
"""
try:
validateArgumentsCreate(taskConf, taskArgumentDefinition, checkInputDset=False)
except WMSpecFactoryException:
# just re-raise it to keep the error message clear
raise
except Exception as ex:
self.raiseValidationException(str(ex))
return
def validateTask(self, taskConf, taskArgumentDefinition):
"""
_validateTask_
Validate the task information against the given
argument description
"""
try:
validateArgumentsCreate(taskConf,
taskArgumentDefinition,
checkInputDset=False)
# Validate GPU-related spec parameters
StdBase.validateGPUSettings(taskConf)
except WMSpecFactoryException:
# just re-raise it to keep the error message clear
raise
except Exception as ex:
self.raiseValidationException(str(ex))
def validateSchema(self, schema):
"""
Since we skip the master validation for Resubmission specs, we better have
some specific validation
"""
if schema.get('OriginalRequestType') == 'Resubmission':
# we cannot validate such schema
return
# load assignment + creation + resubmission creation args definition
argumentDefinition = self.getWorkloadAssignArgs()
parentSpecClass = loadSpecClassByType(schema['OriginalRequestType'])
argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs())
argumentDefinition.update(self.getWorkloadCreateArgs())
try:
validateArgumentsCreate(schema, argumentDefinition)
except Exception as ex:
self.raiseValidationException(str(ex))
def validateSchema(self, schema):
"""
Since we skip the master validation for Resubmission specs, we better have
some specific validation
"""
if schema.get('OriginalRequestType') == 'Resubmission':
# we cannot validate such schema
return
# load assignment + creation + resubmission creation args definition
argumentDefinition = self.getWorkloadAssignArgs()
parentSpecClass = loadSpecClassByType(schema['OriginalRequestType'])
argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs())
argumentDefinition.update(self.getWorkloadCreateArgs())
try:
validateArgumentsCreate(schema, argumentDefinition)
except Exception as ex:
self.raiseValidationException(str(ex))
def validateSchema(self, schema):
"""
_validateSchema_
Check for required fields, and some skim facts
"""
DataProcessing.validateSchema(self, schema)
mainOutputModules = self.validateConfigCacheExists(configID = schema["ConfigCacheID"],
configCacheUrl = schema['ConfigCacheUrl'],
couchDBName = schema["CouchDBName"],
getOutputModules = True).keys()
# Skim facts have to be validated outside the usual master validation
skimSchema = {k: v for (k, v) in schema.iteritems() if k.startswith("Skim")}
skimArguments = self.getSkimArguments()
skimIndex = 1
skimInputs = set()
while "SkimName%s" % skimIndex in schema:
instanceArguments = {}
for argument in skimArguments:
realArg = argument.replace("#N", str(skimIndex))
instanceArguments[realArg] = skimArguments[argument]
try:
validateArgumentsCreate(skimSchema, instanceArguments)
except Exception as ex:
self.raiseValidationException(str(ex))
self.validateConfigCacheExists(configID = schema["Skim%sConfigCacheID" % skimIndex],
configCacheUrl = schema['ConfigCacheUrl'],
couchDBName = schema["CouchDBName"],
getOutputModules=False)
if schema["SkimInput%s" % skimIndex] not in mainOutputModules:
error = "Processing config does not have the following output module: %s." % schema["SkimInput%s" % skimIndex]
self.raiseValidationException(msg = error)
skimInputs.add(schema["SkimInput%s" % skimIndex])
skimIndex += 1
# Validate that the transient output modules are used in a skim task
if "TransientOutputModules" in schema:
diffSet = set(schema["TransientOutputModules"]) - skimInputs
if diffSet:
self.raiseValidationException(msg = "A transient output module was specified but no skim was defined for it")
def validateTask(self, taskConf, taskArgumentDefinition):
"""
_validateTask_
Validate the task information against the given
argument description
"""
msg = validateArgumentsCreate(taskConf, taskArgumentDefinition)
if msg is not None:
self.raiseValidationException(msg)
# Also retrieve the "main" arguments which may be overriden in the task
# Change them all to optional for validation
baseArgs = self.getWorkloadArguments()
validateArgumentsNoOptionalCheck(taskConf, baseArgs)
for arg in baseArgs:
baseArgs[arg]["optional"] = True
msg = validateArgumentsCreate(taskConf, baseArgs)
if msg is not None:
self.raiseValidationException(msg)
return
def validateTask(self, taskConf, taskArgumentDefinition):
"""
_validateTask_
Validate the task information against the given
argument description
"""
msg = validateArgumentsCreate(taskConf, taskArgumentDefinition)
if msg is not None:
self.raiseValidationException(msg)
# Also retrieve the "main" arguments which may be overriden in the task
# Change them all to optional for validation
baseArgs = self.getWorkloadArguments()
validateArgumentsNoOptionalCheck(taskConf, baseArgs)
for arg in baseArgs:
baseArgs[arg]["optional"] = True
msg = validateArgumentsCreate(taskConf, baseArgs)
if msg is not None:
self.raiseValidationException(msg)
return
def validateSchema(self, schema):
"""
Since we skip the master validation for Resubmission specs, we better have
some specific validation
"""
#TODO: for the legacy code if ACDC is created before the updated. remove in next release (Mar 2018)
if schema.get('OriginalRequestType') == 'Resubmission':
# we cannot validate such schema
return
if schema.get("ResubmissionCount", 1) > 1:
# we cannot validate such schema
return
# load assignment + creation + resubmission creation args definition
argumentDefinition = self.getWorkloadAssignArgs()
parentSpecClass = loadSpecClassByType(schema['OriginalRequestType'])
argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs())
argumentDefinition.update(self.getWorkloadCreateArgs())
try:
validateArgumentsCreate(schema, argumentDefinition)
except Exception as ex:
self.raiseValidationException(str(ex))
def validateSchema(self, schema):
"""
Since we skip the master validation for Resubmission specs, we better have
some specific validation
"""
#TODO: for the legacy code if ACDC is created before the updated. remove in next release (Mar 2018)
if schema.get('OriginalRequestType') == 'Resubmission':
# we cannot validate such schema
return
if schema.get("ResubmissionCount", 1) > 1:
# we cannot validate such schema
return
# load assignment + creation + resubmission creation args definition
argumentDefinition = self.getWorkloadAssignArgs()
parentSpecClass = loadSpecClassByType(schema['OriginalRequestType'])
argumentDefinition.update(parentSpecClass.getWorkloadCreateArgs())
argumentDefinition.update(self.getWorkloadCreateArgs())
try:
validateArgumentsCreate(schema, argumentDefinition)
except Exception as ex:
self.raiseValidationException(str(ex))
def validateSchema(self, schema):
"""
_validateSchema_
Check for required fields, and some skim facts
"""
DataProcessing.validateSchema(self, schema)
couchUrl = schema.get("ConfigCacheUrl", None) or schema["CouchURL"]
mainOutputModules = self.validateConfigCacheExists(
configID=schema["ConfigCacheID"],
couchURL=couchUrl,
couchDBName=schema["CouchDBName"],
getOutputModules=True).keys()
# Skim facts have to be validated outside the usual master validation
skimArguments = self.getSkimArguments()
skimIndex = 1
skimInputs = set()
while "SkimName%s" % skimIndex in schema:
instanceArguments = {}
for argument in skimArguments.keys():
realArg = argument.replace("#N", str(skimIndex))
instanceArguments[realArg] = skimArguments[argument]
msg = validateArgumentsCreate(schema, instanceArguments)
if msg is not None:
self.raiseValidationException(msg)
self.validateConfigCacheExists(
configID=schema["Skim%sConfigCacheID" % skimIndex],
couchURL=couchUrl,
couchDBName=schema["CouchDBName"])
if schema["SkimInput%s" % skimIndex] not in mainOutputModules:
error = "Processing config does not have the following output module: %s." % schema[
"SkimInput%s" % skimIndex]
self.raiseValidationException(msg=error)
skimInputs.add(schema["SkimInput%s" % skimIndex])
skimIndex += 1
# Validate that the transient output modules are used in a skim task
if "TransientOutputModules" in schema:
diffSet = set(schema["TransientOutputModules"]) - skimInputs
if diffSet:
self.raiseValidationException(
msg=
"A transient output module was specified but no skim was defined for it"
)
def masterValidation(self, schema):
"""
_masterValidation_
This is validation for global inputs that have to be implemented for
multiple types of workflows in the exact same way.
This uses programatically the definitions in getWorkloadArguments
for type-checking, existence, null tests and the specific validation functions.
Any spec-specific extras are implemented in the overriden validateSchema
"""
# Validate the arguments according to the workload arguments definition
argumentDefinition = self.getWorkloadArguments()
msg = validateArgumentsCreate(schema, argumentDefinition)
if msg is not None:
self.raiseValidationException(msg)
return