def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "StoreResults", "optional": False},
"InputDataset": {"optional": False, "validate": dataset, "null": False},
"ConfigCacheID": {"optional": True, "null": True},
"DataTier": {"default": "USER", "type": str,
"optional": True, "validate": None,
"attr": "dataTier", "null": False},
"PhysicsGroup": {"default": "", "optional": False,
"null": False, "validate": physicsgroup},
"MergedLFNBase": {"default": "/store/results", "type": str,
"optional": True, "validate": None,
"attr": "mergedLFNBase", "null": False},
# site whitelist shouldn't be allowed, but let's make an exception for StoreResults
"SiteWhitelist": {"default": [], "type": makeNonEmptyList, "assign_optional": False,
"validate": lambda x: all([cmsname(y) for y in x])},
"BlockBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockBlacklist", "null": False},
"BlockWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockWhitelist", "null": False},
"RunBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runBlacklist", "null": False},
"RunWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runWhitelist", "null": False}}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "Express"},
"ConfigCacheID": {"optional": True, "null": True},
"Scenario": {"optional": False, "attr": "procScenario"},
"RecoCMSSWVersion": {"validate": lambda x: x in releases(),
"optional": False, "attr": "recoFrameworkVersion"},
"RecoScramArch": {"validate": lambda x: all([y in architectures() for y in x]),
"optional": False, "type": makeNonEmptyList},
"GlobalTag": {"optional": False},
"GlobalTagTransaction": {"optional": False},
"ProcessingString": {"default": "", "validate": procstringT0},
"StreamName": {"optional": False},
"SpecialDataset": {"optional": False},
"AlcaHarvestTimeout": {"type": int, "optional": False},
"AlcaHarvestDir": {"optional": False, "null": True},
"AlcaSkims": {"type": makeList, "optional": False},
"DQMSequences": {"type": makeList, "attr": "dqmSequences", "optional": False},
"Outputs": {"type": makeList, "optional": False},
"MaxInputRate": {"type": int, "optional": False},
"MaxInputEvents": {"type": int, "optional": False},
"MaxInputSize": {"type": int, "optional": False},
"MaxInputFiles": {"type": int, "optional": False},
"MaxLatency": {"type": int, "optional": False},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
"""
Some default values set for testing purposes
"""
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "MonteCarlo", "optional": False},
"PrimaryDataset": {"optional": False, "validate": primdataset,
"attr": "inputPrimaryDataset", "null": False},
"Seeding": {"default": "AutomaticSeeding", "null": False,
"validate": lambda x: x in ["ReproducibleSeeding", "AutomaticSeeding"]},
"FilterEfficiency": {"default": 1.0, "type": float, "null": False,
"validate": lambda x: x > 0.0},
"RequestNumEvents": {"type": int, "null": False,
"optional": False, "validate": lambda x: x > 0},
"FirstEvent": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False},
"FirstLumi": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False},
"MCPileup": {"validate": dataset, "attr": "mcPileup", "null": True},
"DataPileup": {"validate": dataset, "attr": "dataPileup", "null": True},
"SplittingAlgo": {"default": "EventBased", "null": False,
"validate": lambda x: x in ["EventBased"],
"attr": "prodJobSplitAlgo"},
"DeterministicPileup": {"default": False, "type": strToBool, "null": False},
"EventsPerJob": {"type": int, "validate": lambda x: x > 0, "null": True},
"EventsPerLumi": {"type": int, "validate": lambda x: x > 0, "null": True},
"LheInputFiles": {"default": False, "type": strToBool, "null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"InputDataset": {"optional": False, "validate": dataset, "null": False},
"Scenario": {"optional": True, "null": True, "attr": "procScenario"},
"PrimaryDataset": {"optional": True, "validate": primdataset,
"attr": "inputPrimaryDataset", "null": True},
"RunBlacklist": {"default": [], "type": makeList, "null": False,
"validate": lambda x: all([int(y) > 0 for y in x])},
"RunWhitelist": {"default": [], "type": makeList, "null": False,
"validate": lambda x: all([int(y) > 0 for y in x])},
"BlockBlacklist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"BlockWhitelist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"SplittingAlgo": {"default": "EventAwareLumiBased", "null": False,
"validate": lambda x: x in ["EventBased", "LumiBased",
"EventAwareLumiBased", "FileBased"],
"attr": "procJobSplitAlgo"},
"EventsPerJob": {"type": int, "validate": lambda x: x > 0, "null": True},
"LumisPerJob": {"default": 8, "type": int, "null": False,
"validate": lambda x: x > 0},
"FilesPerJob": {"default": 1, "type": int, "null": False,
"validate": lambda x: x > 0}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "StoreResults", "optional": False},
"InputDataset": {"optional": False, "validate": dataset, "null": False},
"ConfigCacheID": {"optional": True, "null": True},
"DataTier": {"default": "USER", "type": str,
"optional": True, "validate": None,
"attr": "dataTier", "null": False},
"PhysicsGroup": {"default": "", "optional": False,
"null": False, "validate": physicsgroup},
"MergedLFNBase": {"default": "/store/results", "type": str,
"optional": True, "validate": None,
"attr": "mergedLFNBase", "null": False},
"BlockBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockBlacklist", "null": False},
"BlockWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockWhitelist", "null": False},
"RunBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runBlacklist", "null": False},
"RunWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runWhitelist", "null": False}}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"InputDataset": {"optional": False, "validate": dataset, "null": False},
"Scenario": {"optional": True, "null": True, "attr": "procScenario"},
"PrimaryDataset": {"optional": True, "validate": primdataset,
"attr": "inputPrimaryDataset", "null": True},
"RunBlacklist": {"default": [], "type": makeList, "null": False,
"validate": lambda x: all([int(y) > 0 for y in x])},
"RunWhitelist": {"default": [], "type": makeList, "null": False,
"validate": lambda x: all([int(y) > 0 for y in x])},
"BlockBlacklist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"BlockWhitelist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"SplittingAlgo": {"default": "EventAwareLumiBased", "null": False,
"validate": lambda x: x in ["EventBased", "LumiBased",
"EventAwareLumiBased", "FileBased"],
"attr": "procJobSplitAlgo"},
"EventsPerJob": {"type": int, "validate": lambda x: x > 0, "null": True},
"LumisPerJob": {"default": 8, "type": int, "null": False,
"validate": lambda x: x > 0},
"FilesPerJob": {"default": 1, "type": int, "null": False,
"validate": lambda x: x > 0}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
"""
Some default values set for testing purposes
"""
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "MonteCarlo", "optional": False},
"PrimaryDataset": {"optional": False, "validate": primdataset,
"attr": "inputPrimaryDataset", "null": False},
"Seeding": {"default": "AutomaticSeeding", "null": False,
"validate": lambda x: x in ["ReproducibleSeeding", "AutomaticSeeding"]},
"FilterEfficiency": {"default": 1.0, "type": float, "null": False,
"validate": lambda x: x > 0.0},
"RequestNumEvents": {"type": int, "null": False,
"optional": False, "validate": lambda x: x > 0},
"FirstEvent": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False},
"FirstLumi": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False},
"MCPileup": {"validate": dataset, "attr": "mcPileup", "null": True},
"DataPileup": {"validate": dataset, "attr": "dataPileup", "null": True},
"SplittingAlgo": {"default": "EventBased", "null": False,
"validate": lambda x: x in ["EventBased"],
"attr": "prodJobSplitAlgo"},
"DeterministicPileup": {"default": False, "type": strToBool, "null": False},
"EventsPerJob": {"type": int, "validate": lambda x: x > 0, "null": True},
"EventsPerLumi": {"type": int, "validate": lambda x: x > 0, "null": True},
"LheInputFiles": {"default": False, "type": strToBool, "null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "StepChain",
"optional": False
},
"Step1": {
"default": {},
"optional": False,
"type": dict
},
# ConfigCacheID is not used in the main dict for StepChain
"ConfigCacheID": {
"optional": True,
"null": True
},
"DeterministicPileup": {
"default": False,
"type": strToBool,
"optional": True,
"null": False
},
"PrimaryDataset": {
"null": True,
"validate": primdataset
},
"StepChain": {
"default": 1,
"type": int,
"null": False,
"optional": False,
"validate": lambda x: x > 0
},
"ChainParentageMap": {
"default": {},
"type": dict
},
"FirstEvent": {
"default": 1,
"type": int,
"validate": lambda x: x > 0,
"null": False
},
"FirstLumi": {
"default": 1,
"type": int,
"validate": lambda x: x > 0,
"null": False
},
"ParentageResolved": {
"default": False,
"type": strToBool,
"null": False
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "TaskChain",
"optional": False
},
"Task1": {
"default": {},
"optional": False,
"type": dict
},
# ConfigCacheID is not used in the main dict for TaskChain
"ConfigCacheID": {
"optional": True,
"null": True
},
"IgnoredOutputModules": {
"default": [],
"type": makeList,
"null": False
},
"TaskChain": {
"default": 1,
"type": int,
"optional": False,
"validate": lambda x: x > 0,
"attr": "taskChain",
"null": False
},
"ChainParentageMap": {
"default": {},
"type": dict
},
"FirstEvent": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "firstEvent",
"null": False
},
"FirstLumi": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "firstLumi",
"null": False
}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def validate_request_priority(reqArgs):
"""
Validate the RequestPriority argument against its definition
in StdBase
:param reqArgs: dictionary of user request arguments
:return: nothing, but raises an exception in case of an invalid value
"""
if 'RequestPriority' in reqArgs:
reqPrioDefin = StdBase.getWorkloadCreateArgs()['RequestPriority']
if not isinstance(reqArgs['RequestPriority'], reqPrioDefin['type']):
msg = "RequestPriority must be of integer type, not: {}".format(
type(reqArgs['RequestPriority']))
raise InvalidSpecParameterValue(msg)
if reqPrioDefin['validate'](reqArgs['RequestPriority']) is False:
raise InvalidSpecParameterValue(
"RequestPriority must be an integer between 0 and 999999")
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "StepChain", "optional": False},
"Step1": {"default": {}, "optional": False, "type": dict},
# ConfigCacheID is not used in the main dict for StepChain
"ConfigCacheID": {"optional": True, "null": True},
"DeterministicPileup": {"default": False, "type": strToBool, "optional": True, "null": False},
"PrimaryDataset": {"null": True, "validate": primdataset},
"StepChain": {"default": 1, "type": int, "null": False,
"optional": False, "validate": lambda x: x > 0},
"FirstEvent": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False},
"FirstLumi": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "TaskChain", "optional": False},
"Task1": {"default": {}, "optional": False, "type": dict},
# ConfigCacheID is not used in the main dict for TaskChain
"ConfigCacheID": {"optional": True, "null": True},
"IgnoredOutputModules": {"default": [], "type": makeList, "null": False},
"TaskChain": {"default": 1, "type": int,
"optional": False, "validate": lambda x: x > 0,
"attr": "taskChain", "null": False},
"FirstEvent": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstEvent", "null": False},
"FirstLumi": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstLumi", "null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "Repack"},
"ConfigCacheID": {"optional": True, "null": True},
"Scenario": {"default": "fake", "attr": "procScenario"},
"GlobalTag": {"default": "fake"},
"ProcessingString": {"default": "", "validate": procstringT0},
"Outputs": {"type": makeList, "optional": False},
"MaxSizeSingleLumi": {"type": int, "optional": False},
"MaxSizeMultiLumi": {"type": int, "optional": False},
"MaxInputEvents": {"type": int, "optional": False},
"MaxInputFiles": {"type": int, "optional": False},
"MaxLatency": {"type": int, "optional": False},
"MinInputSize": {"type": int, "optional": False},
"MaxInputSize": {"type": int, "optional": False},
"MaxEdmSize": {"type": int, "optional": False},
"MaxOverSize": {"type": int, "optional": False},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "Repack"
},
"ConfigCacheID": {
"optional": True,
"null": True
},
"Scenario": {
"default": "fake",
"attr": "procScenario"
},
"GlobalTag": {
"default": "fake"
},
"ProcessingString": {
"default": "",
"validate": procstringT0
},
"Outputs": {
"type": makeList,
"optional": False
},
"MaxSizeSingleLumi": {
"type": int,
"optional": False
},
"MaxSizeMultiLumi": {
"type": int,
"optional": False
},
"MaxInputEvents": {
"type": int,
"optional": False
},
"MaxInputFiles": {
"type": int,
"optional": False
},
"MaxLatency": {
"type": int,
"optional": False
},
"MinInputSize": {
"type": int,
"optional": False
},
"MaxInputSize": {
"type": int,
"optional": False
},
"MaxEdmSize": {
"type": int,
"optional": False
},
"MaxOverSize": {
"type": int,
"optional": False
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "StepChain",
"optional": False
},
"Step1": {
"default": {},
"optional": False,
"type": dict
},
# ConfigCacheID is not used in the main dict for StepChain
"ConfigCacheID": {
"optional": True,
"null": True
},
"DeterministicPileup": {
"default": False,
"type": strToBool,
"optional": True,
"null": False
},
"PrimaryDataset": {
"null": True,
"validate": primdataset
},
"StepChain": {
"default": 1,
"type": int,
"null": False,
"optional": False,
"validate": lambda x: x > 0
},
"ChainParentageMap": {
"default": {},
"type": dict
},
"FirstEvent": {
"default": 1,
"type": int,
"validate": lambda x: x > 0,
"null": False
},
"FirstLumi": {
"default": 1,
"type": int,
"validate": lambda x: x > 0,
"null": False
},
"ParentageResolved": {
"default": False,
"type": strToBool,
"null": False
},
### Override StdBase parameter definition
"TimePerEvent": {
"default": 12.0,
"type": float,
"validate": lambda x: x > 0
},
"Memory": {
"default": 2300.0,
"type": float,
"validate": lambda x: x > 0
},
"Multicore": {
"default": 1,
"type": int,
"validate": checkMemCore
},
"EventStreams": {
"type": int,
"null": True,
"default": 0,
"validate": checkEventStreams
}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "StoreResults",
"optional": False
},
"InputDataset": {
"optional": False,
"validate": dataset,
"null": False
},
"CmsPath": {
"default": "/tmp",
"type": str,
"optional": False,
"validate": None,
"attr": "cmsPath",
"null": False
},
"DataTier": {
"default": "USER",
"type": str,
"optional": True,
"validate": None,
"attr": "dataTier",
"null": False
},
"MergedLFNBase": {
"default": "/store/results",
"type": str,
"optional": True,
"validate": None,
"attr": "mergedLFNBase",
"null": False
},
"BlockBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"attr": "blockBlacklist",
"null": False
},
"BlockWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"attr": "blockWhitelist",
"null": False
},
"RunBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runBlacklist",
"null": False
},
"RunWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runWhitelist",
"null": False
}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "Express"
},
"ConfigCacheID": {
"optional": True,
"null": True
},
"Scenario": {
"optional": False,
"attr": "procScenario"
},
"RecoCMSSWVersion": {
"validate": lambda x: x in releases(),
"optional": False,
"attr": "recoFrameworkVersion"
},
"RecoScramArch": {
"validate": lambda x: all([y in architectures() for y in x]),
"optional": False,
"type": makeNonEmptyList
},
"GlobalTag": {
"optional": False
},
"GlobalTagTransaction": {
"optional": False
},
"ProcessingString": {
"default": "",
"validate": procstringT0
},
"StreamName": {
"optional": False
},
"SpecialDataset": {
"optional": False
},
"AlcaHarvestTimeout": {
"type": int,
"optional": False
},
"AlcaHarvestDir": {
"optional": False,
"null": True
},
"AlcaSkims": {
"type": makeList,
"optional": False
},
"DQMSequences": {
"type": makeList,
"attr": "dqmSequences",
"optional": False
},
"Outputs": {
"type": makeList,
"optional": False
},
"MaxInputRate": {
"type": int,
"optional": False
},
"MaxInputEvents": {
"type": int,
"optional": False
},
"MaxInputSize": {
"type": int,
"optional": False
},
"MaxInputFiles": {
"type": int,
"optional": False
},
"MaxLatency": {
"type": int,
"optional": False
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "TaskChain",
"optional": False
},
"Task1": {
"default": {},
"optional": False,
"type": dict
},
# ConfigCacheID is not used in the main dict for TaskChain
"ConfigCacheID": {
"optional": True,
"null": True
},
"IgnoredOutputModules": {
"default": [],
"type": makeList,
"null": False
},
"TaskChain": {
"default": 1,
"type": int,
"optional": False,
"validate": lambda x: x > 0,
"attr": "taskChain",
"null": False
},
"ChainParentageMap": {
"default": {},
"type": dict
},
"FirstEvent": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "firstEvent",
"null": False
},
"FirstLumi": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "firstLumi",
"null": False
},
### Override StdBase parameter definition
"TimePerEvent": {
"default": 12.0,
"type": float,
"validate": checkTimePerEvent
},
"Memory": {
"default": 2300.0,
"type": float,
"validate": checkMemCore
},
"Multicore": {
"default": 1,
"type": int,
"validate": checkMemCore
},
"EventStreams": {
"type": int,
"null": True,
"default": 0,
"validate": checkEventStreams
},
# no need for workload-level defaults, if task-level default is provided
"RequiresGPU": {
"default": None,
"null": True,
"validate": lambda x: x in
("forbidden", "optional", "required")
},
"GPUParams": {
"default": json.dumps(None),
"validate": gpuParameters
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
