def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
specArgs = {
"RequestType": {
"default": "StepChain",
"optional": False
},
"GlobalTag": {
"type": str,
"optional": False
},
"CouchURL": {
"type": str,
"optional": False,
"validate": couchurl
},
"PrimaryDataset": {
"default": None,
"type": str,
"validate": primdataset,
"null": False
},
"CouchDBName": {
"type": str,
"optional": False,
"validate": identifier
},
"ConfigCacheUrl": {
"type": str,
"optional": True,
"null": True
},
"StepChain": {
"default": 1,
"type": int,
"optional": False,
"validate": lambda x: x > 0,
"attr": "stepChain",
"null": False
},
"FirstEvent": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "firstEvent",
"null": False
},
"FirstLumi": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "firstLumi",
"null": False
}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"InputDataset": {"optional": False, "validate": dataset, "null": False},
"Scenario": {"optional": True, "null": True, "attr": "procScenario"},
"PrimaryDataset": {"optional": True, "validate": primdataset,
"attr": "inputPrimaryDataset", "null": True},
"RunBlacklist": {"default": [], "type": makeList, "null": False,
"validate": lambda x: all([int(y) > 0 for y in x])},
"RunWhitelist": {"default": [], "type": makeList, "null": False,
"validate": lambda x: all([int(y) > 0 for y in x])},
"BlockBlacklist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"BlockWhitelist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"SplittingAlgo": {"default": "EventAwareLumiBased", "null": False,
"validate": lambda x: x in ["EventBased", "LumiBased",
"EventAwareLumiBased", "FileBased"],
"attr": "procJobSplitAlgo"},
"EventsPerJob": {"type": int, "validate": lambda x: x > 0, "null": True},
"LumisPerJob": {"default": 8, "type": int, "null": False,
"validate": lambda x: x > 0},
"FilesPerJob": {"default": 1, "type": int, "null": False,
"validate": lambda x: x > 0}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
"""
If arg is not specifyed, automatically set by following default value
- default: None
- type: str
- optional: True
- assign_optional: True
- validate: None
- attr: change first letter to lower case
- null: False
"""
baseArgs = StdBase.getWorkloadArguments()
## specArgs = {"Outputs" : {"default" : {}, "type" : dict,
## "optional" : False, "validate" : None,
## "attr" : "outputs", "null" : False},
specArgs = {"RequestType" : {"default" : "Express"},
"Scenario" : {"optional" : False, "attr" : "procScenario"},
"RecoCMSSWVersion" : {"optional" : False, "validate" : cmsswversion,
"attr" : "recoFrameworkVersion", "null" : True},
"RecoScramArch" : {"optional" : False, "null" : True},
"GlobalTag" : {"optional" : False},
"GlobalTagTransaction" : {"optional" : False},
"StreamName" : {"optional" : False},
"SpecialDataset" : {"optional" : False},
"AlcaHarvestTimeout" : {"type" : int, "optional" : False},
"AlcaHarvestDir" : {"optional" : False, "null" : True},
"AlcaSkims" : {"type" : makeList, "optional" : False},
"DqmSequences" : {"type" : makeList, "optional" : False},
"BlockCloseDelay" : {"type" : int, "optional" : False,
"validate" : lambda x : x > 0},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {"RequestType": {"default": "DQMHarvest"},
"InputDataset": {"default": None, "optional": False,
"validate": dataset},
"UnmergedLFNBase": {"default": "/store/unmerged"},
"MergedLFNBase": {"default": "/store/data"},
"MinMergeSize": {"default": 2 * 1024 * 1024 * 1024, "type": int,
"validate": lambda x: x > 0},
"MaxMergeSize": {"default": 4 * 1024 * 1024 * 1024, "type": int,
"validate": lambda x: x > 0},
"MaxMergeEvents": {"default": 100000, "type": int,
"validate": lambda x: x > 0},
"BlockBlacklist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"BlockWhitelist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"RunBlacklist": {"default": [], "type": makeList,
"validate": lambda x: all([int(y) > 0 for y in x])},
"RunWhitelist": {"default": [], "type": makeList,
"validate": lambda x: all([int(y) > 0 for y in x])}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "Express"},
"ConfigCacheID": {"optional": True, "null": True},
"Scenario": {"optional": False, "attr": "procScenario"},
"RecoCMSSWVersion": {"validate": lambda x: x in releases(),
"optional": False, "attr": "recoFrameworkVersion"},
"RecoScramArch": {"validate": lambda x: all([y in architectures() for y in x]),
"optional": False, "type": makeNonEmptyList},
"GlobalTag": {"optional": False},
"GlobalTagTransaction": {"optional": False},
"ProcessingString": {"default": "", "validate": procstringT0},
"StreamName": {"optional": False},
"SpecialDataset": {"optional": False},
"AlcaHarvestTimeout": {"type": int, "optional": False},
"AlcaHarvestDir": {"optional": False, "null": True},
"AlcaSkims": {"type": makeList, "optional": False},
"DQMSequences": {"type": makeList, "attr": "dqmSequences", "optional": False},
"Outputs": {"type": makeList, "optional": False},
"MaxInputRate": {"type": int, "optional": False},
"MaxInputEvents": {"type": int, "optional": False},
"MaxInputSize": {"type": int, "optional": False},
"MaxInputFiles": {"type": int, "optional": False},
"MaxLatency": {"type": int, "optional": False},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
"""
If arg is not specifyed, automatically set by following default value
- default: None
- type: str
- optional: True
- assign_optional: True
- validate: None
- attr: change first letter to lower case
- null: False
"""
baseArgs = StdBase.getWorkloadArguments()
## specArgs = {"Outputs" : {"default" : {}, "type" : dict,
## "optional" : False, "validate" : None,
## "attr" : "outputs", "null" : False},
specArgs = {
"RequestType": {
"default": "Repack"
},
"Scenario": {
"default": "fake",
"attr": "procScenario"
},
"GlobalTag": {
"default": "fake"
},
"BlockCloseDelay": {
"type": int,
"optional": False,
"validate": lambda x: x > 0,
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
"""
Some default values set for testing purposes
"""
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "MonteCarlo", "optional": False},
"PrimaryDataset": {"optional": False, "validate": primdataset,
"attr": "inputPrimaryDataset", "null": False},
"Seeding": {"default": "AutomaticSeeding", "null": False,
"validate": lambda x: x in ["ReproducibleSeeding", "AutomaticSeeding"]},
"FilterEfficiency": {"default": 1.0, "type": float, "null": False,
"validate": lambda x: x > 0.0},
"RequestNumEvents": {"type": int, "null": False,
"optional": False, "validate": lambda x: x > 0},
"FirstEvent": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False},
"FirstLumi": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False},
"MCPileup": {"validate": dataset, "attr": "mcPileup", "null": True},
"DataPileup": {"validate": dataset, "attr": "dataPileup", "null": True},
"SplittingAlgo": {"default": "EventBased", "null": False,
"validate": lambda x: x in ["EventBased"],
"attr": "prodJobSplitAlgo"},
"DeterministicPileup": {"default": False, "type": strToBool, "null": False},
"EventsPerJob": {"type": int, "validate": lambda x: x > 0, "null": True},
"EventsPerLumi": {"type": int, "validate": lambda x: x > 0, "null": True},
"LheInputFiles": {"default": False, "type": strToBool, "null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "StoreResults", "optional": False},
"InputDataset": {"optional": False, "validate": dataset, "null": False},
"ConfigCacheID": {"optional": True, "null": True},
"DataTier": {"default": "USER", "type": str,
"optional": True, "validate": None,
"attr": "dataTier", "null": False},
"PhysicsGroup": {"default": "", "optional": False,
"null": False, "validate": physicsgroup},
"MergedLFNBase": {"default": "/store/results", "type": str,
"optional": True, "validate": None,
"attr": "mergedLFNBase", "null": False},
"BlockBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockBlacklist", "null": False},
"BlockWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockWhitelist", "null": False},
"RunBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runBlacklist", "null": False},
"RunWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runWhitelist", "null": False}}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {"RequestType": {"default": "TaskChain", "optional": False,
"attr": "requestType"},
# ConfigCacheID is not mandatory in the main dict for TaskChain
"ConfigCacheID": {"optional": True, "null": True},
"CouchURL": {"default": "http://localhost:5984", "type": str,
"optional": False, "validate": couchurl,
"attr": "couchURL", "null": False},
"CouchDBName": {"default": "dp_configcache", "type": str,
"optional": False, "validate": identifier,
"attr": "couchDBName", "null": False},
"ConfigCacheUrl": {"default": None, "type": str,
"optional": True, "validate": None,
"attr": "configCacheUrl", "null": True},
"IgnoredOutputModules": {"default": [], "type": makeList,
"optional": True, "validate": None,
"attr": "ignoredOutputModules", "null": False},
"TaskChain": {"default": 1, "type": int,
"optional": False, "validate": lambda x: x > 0,
"attr": "taskChain", "null": False},
"FirstEvent": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstEvent", "null": False},
"FirstLumi": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstLumi", "null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getChainCreateArgs(firstTask=False, generator=False):
"""
_getChainCreateArgs_
This represents the authoritative list of request arguments that are
allowed in each chain (Step/Task) of chained request, during request creation.
Additional especific arguments must be defined inside each spec class.
For more information on how these arguments are built, please have a look
at the docstring for getWorkloadCreateArgs.
"""
baseArgs = StdBase.getChainCreateArgs(firstTask, generator)
arguments = {
'InputStep': {
'default': None,
'null': False,
'optional': firstTask
},
'StepName': {
'null': False,
'optional': False,
'validate': taskStepName
},
'PrimaryDataset': {
'default': None,
'optional': True,
'validate': primdataset,
'null': False
}
}
baseArgs.update(arguments)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {"RequestType": {"default": "TaskChain", "optional": False,
"attr": "requestType"},
"GlobalTag": {"default": "GT_TC_V1", "type": str,
"optional": False, "validate": None,
"attr": "globalTag", "null": False},
"CouchURL": {"default": "http://localhost:5984", "type": str,
"optional": False, "validate": couchurl,
"attr": "couchURL", "null": False},
"CouchDBName": {"default": "dp_configcache", "type": str,
"optional": False, "validate": identifier,
"attr": "couchDBName", "null": False},
"ConfigCacheUrl": {"default": None, "type": str,
"optional": True, "validate": None,
"attr": "configCacheUrl", "null": True},
"IgnoredOutputModules": {"default": [], "type": makeList,
"optional": True, "validate": None,
"attr": "ignoredOutputModules", "null": False},
"TaskChain": {"default": 1, "type": int,
"optional": False, "validate": lambda x: x > 0,
"attr": "taskChain", "null": False},
"FirstEvent": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstEvent", "null": False},
"FirstLumi": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstLumi", "null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {"RequestType": {"default": "DQMHarvest"},
"InputDataset": {"default": None, "optional": False,
"validate": dataset},
"ConfigCacheID": {"optional": True, "validate": None, "null": True},
"UnmergedLFNBase": {"default": "/store/unmerged"},
"MergedLFNBase": {"default": "/store/data"},
"MinMergeSize": {"default": 2 * 1024 * 1024 * 1024, "type": int,
"validate": lambda x: x > 0},
"MaxMergeSize": {"default": 4 * 1024 * 1024 * 1024, "type": int,
"validate": lambda x: x > 0},
"MaxMergeEvents": {"default": 100000, "type": int,
"validate": lambda x: x > 0},
"BlockBlacklist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"BlockWhitelist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"RunBlacklist": {"default": [], "type": makeList,
"validate": lambda x: all([int(y) > 0 for y in x])},
"RunWhitelist": {"default": [], "type": makeList,
"validate": lambda x: all([int(y) > 0 for y in x])}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
"""
If arg is not specifyed, automatically set by following default value
- default: None
- type: str
- optional: True
- assign_optional: True
- validate: None
- attr: change first letter to lower case
- null: False
"""
baseArgs = StdBase.getWorkloadArguments()
## specArgs = {"Outputs" : {"default" : {}, "type" : dict,
## "optional" : False, "validate" : None,
## "attr" : "outputs", "null" : False},
specArgs = {"RequestType" : {"default" : "Repack"},
"Scenario" : {"default" : "fake", "attr" : "procScenario"},
"GlobalTag" : {"default" : "fake"},
"BlockCloseDelay" : {"type" : int, "optional" : False,
"validate" : lambda x : x > 0,
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
"""
Some default values set for testing purposes
"""
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "MonteCarlo", "optional": False},
"PrimaryDataset": {"optional": False, "validate": primdataset,
"attr": "inputPrimaryDataset", "null": False},
"Seeding": {"default": "AutomaticSeeding", "null": False,
"validate": lambda x: x in ["ReproducibleSeeding", "AutomaticSeeding"]},
"FilterEfficiency": {"default": 1.0, "type": float, "null": False,
"validate": lambda x: x > 0.0},
"RequestNumEvents": {"type": int, "null": False,
"optional": False, "validate": lambda x: x > 0},
"FirstEvent": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False},
"FirstLumi": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False},
"MCPileup": {"validate": dataset, "attr": "mcPileup", "null": True},
"DataPileup": {"validate": dataset, "attr": "dataPileup", "null": True},
"SplittingAlgo": {"default": "EventBased", "null": False,
"validate": lambda x: x in ["EventBased"],
"attr": "prodJobSplitAlgo"},
"DeterministicPileup": {"default": False, "type": strToBool, "null": False},
"EventsPerJob": {"type": int, "validate": lambda x: x > 0, "null": True},
"EventsPerLumi": {"type": int, "validate": lambda x: x > 0, "null": True},
"LheInputFiles": {"default": False, "type": strToBool, "null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
"""
If arg is not specifyed, automatically set by following default value
- default: None
- type: str
- optional: True
- assign_optional: True
- validate: None
- attr: change first letter to lower case
- null: False
"""
baseArgs = StdBase.getWorkloadArguments()
specArgs = {"RequestType": {"default" : "Express"},
"Scenario": {"optional" : False, "attr" : "procScenario"},
"RecoCMSSWVersion": {"optional" : False, "validate" : cmsswversion,
"attr" : "recoFrameworkVersion", "null" : True},
"RecoScramArch": {"optional" : False, "null" : True},
"GlobalTag": {"optional" : False},
"GlobalTagTransaction": {"optional" : False},
"StreamName": {"optional" : False},
"SpecialDataset": {"optional" : False},
"AlcaHarvestTimeout": {"type" : int, "optional" : False},
"AlcaHarvestDir": {"optional" : False, "null" : True},
"AlcaSkims": {"type" : makeList, "optional" : False},
"DQMSequences": {"type" : makeList, "attr" : "dqmSequences", "optional" : False},
"BlockCloseDelay": {"type" : int, "optional" : False,
"validate" : lambda x : x > 0
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"InputDataset": {"optional": False, "validate": dataset, "null": False},
"Scenario": {"optional": True, "null": True, "attr": "procScenario"},
"PrimaryDataset": {"optional": True, "validate": primdataset,
"attr": "inputPrimaryDataset", "null": True},
"RunBlacklist": {"default": [], "type": makeList, "null": False,
"validate": lambda x: all([int(y) > 0 for y in x])},
"RunWhitelist": {"default": [], "type": makeList, "null": False,
"validate": lambda x: all([int(y) > 0 for y in x])},
"BlockBlacklist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"BlockWhitelist": {"default": [], "type": makeList,
"validate": lambda x: all([block(y) for y in x])},
"SplittingAlgo": {"default": "EventAwareLumiBased", "null": False,
"validate": lambda x: x in ["EventBased", "LumiBased",
"EventAwareLumiBased", "FileBased"],
"attr": "procJobSplitAlgo"},
"EventsPerJob": {"type": int, "validate": lambda x: x > 0, "null": True},
"LumisPerJob": {"default": 8, "type": int, "null": False,
"validate": lambda x: x > 0},
"FilesPerJob": {"default": 1, "type": int, "null": False,
"validate": lambda x: x > 0}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "StoreResults", "optional": False},
"InputDataset": {"optional": False, "validate": dataset, "null": False},
"ConfigCacheID": {"optional": True, "null": True},
"DataTier": {"default": "USER", "type": str,
"optional": True, "validate": None,
"attr": "dataTier", "null": False},
"PhysicsGroup": {"default": "", "optional": False,
"null": False, "validate": physicsgroup},
"MergedLFNBase": {"default": "/store/results", "type": str,
"optional": True, "validate": None,
"attr": "mergedLFNBase", "null": False},
# site whitelist shouldn't be allowed, but let's make an exception for StoreResults
"SiteWhitelist": {"default": [], "type": makeNonEmptyList, "assign_optional": False,
"validate": lambda x: all([cmsname(y) for y in x])},
"BlockBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockBlacklist", "null": False},
"BlockWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"attr": "blockWhitelist", "null": False},
"RunBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runBlacklist", "null": False},
"RunWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runWhitelist", "null": False}}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
specArgs = {
"RequestType": {
"default": "Resubmission"
},
"InitialTaskPath": {
"default": "/SomeRequest/Task1",
"optional": False,
"validate": lambda x: len(x.split('/')) > 2
},
"ACDCServer": {
"default": "https://cmsweb.cern.ch/couchdb",
"validate": couchurl,
"attr": "acdcServer"
},
"ACDCDatabase": {
"default": "acdcserver",
"validate": identifier,
"attr": "acdcDatabase"
},
"CollectionName": {
"default": None,
"null": True
},
"IgnoredOutputModules": {
"default": [],
"type": makeList
}
}
StdBase.setDefaultArgumentsProperty(specArgs)
return specArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "StepChain",
"optional": False
},
"Step1": {
"default": {},
"optional": False,
"type": dict
},
# ConfigCacheID is not used in the main dict for StepChain
"ConfigCacheID": {
"optional": True,
"null": True
},
"DeterministicPileup": {
"default": False,
"type": strToBool,
"optional": True,
"null": False
},
"PrimaryDataset": {
"null": True,
"validate": primdataset
},
"StepChain": {
"default": 1,
"type": int,
"null": False,
"optional": False,
"validate": lambda x: x > 0
},
"ChainParentageMap": {
"default": {},
"type": dict
},
"FirstEvent": {
"default": 1,
"type": int,
"validate": lambda x: x > 0,
"null": False
},
"FirstLumi": {
"default": 1,
"type": int,
"validate": lambda x: x > 0,
"null": False
},
"ParentageResolved": {
"default": False,
"type": strToBool,
"null": False
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadAssignArgs():
baseArgs = StdBase.getWorkloadAssignArgs()
specArgs = {
"ChainParentageMap": {"default": {}, "type": dict},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadAssignArgs():
baseArgs = StdBase.getWorkloadAssignArgs()
specArgs = {
"ChainParentageMap": {"default": {}, "type": dict},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getChainCreateArgs(firstTask=False, generator=False):
"""
_getChainCreateArgs_
Each task dictionary specifies its own set of arguments
that need to be validated as well, most of them are already
defined in StdBase.getWorkloadCreateArgs and those do not appear here
since they are all optional. Here only new arguments are listed.
"""
baseArgs = StdBase.getChainCreateArgs(firstTask, generator)
arguments = {
"TaskName": {
"optional": False,
"null": False,
'validate': taskStepName
},
"InputTask": {
"default": None,
"optional": firstTask,
"null": False
},
"TransientOutputModules": {
"default": [],
"type": makeList,
"optional": True,
"null": False
},
"DeterministicPileup": {
"default": False,
"type": strToBool,
"optional": True,
"null": False
},
"GlobalTag": {
"type": str,
"optional": True
},
"TimePerEvent": {
"type": float,
"optional": True,
"validate": lambda x: x > 0
},
"SizePerEvent": {
"type": float,
"optional": True,
"validate": lambda x: x > 0
},
'PrimaryDataset': {
'default': None,
'optional': not generator,
'validate': primdataset,
'null': False
},
}
baseArgs.update(arguments)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadAssignArgs():
baseArgs = StdBase.getWorkloadAssignArgs()
specArgs = {
"Override": {"default": {"eos-lfn-prefix": "root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/Repack"},
"type": dict},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "TaskChain",
"optional": False
},
"Task1": {
"default": {},
"optional": False,
"type": dict
},
# ConfigCacheID is not used in the main dict for TaskChain
"ConfigCacheID": {
"optional": True,
"null": True
},
"IgnoredOutputModules": {
"default": [],
"type": makeList,
"null": False
},
"TaskChain": {
"default": 1,
"type": int,
"optional": False,
"validate": lambda x: x > 0,
"attr": "taskChain",
"null": False
},
"ChainParentageMap": {
"default": {},
"type": dict
},
"FirstEvent": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "firstEvent",
"null": False
},
"FirstLumi": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "firstLumi",
"null": False
}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
specArgs = {"RequestType" : {"default" : "Resubmission"},
"InitialTaskPath" : {"default" : "/SomeRequest/Task1", "optional": False,
"validate": lambda x: len(x.split('/')) > 2},
"ACDCServer" : {"default" : "https://cmsweb.cern.ch/couchdb", "validate" : couchurl,
"attr" : "acdcServer"},
"ACDCDatabase" : {"default" : "acdcserver", "validate" : identifier,
"attr" : "acdcDatabase"},
"CollectionName" : {"default" : None, "null" : True},
"IgnoredOutputModules" : {"default" : [], "type" : makeList}}
StdBase.setDefaultArgumentsProperty(specArgs)
return specArgs
def getWorkloadCreateArgs():
specArgs = {
"RequestType": {
"default": "Resubmission"
},
"ResubmissionCount": {
"default": 1,
"type": int
},
"OriginalRequestType": {
"null": False
},
"OriginalRequestName": {
"null": False
},
"InitialTaskPath": {
"optional": False,
"validate": lambda x: len(x.split('/')) > 2
},
"ACDCServer": {
"default": "https://cmsweb.cern.ch/couchdb",
"validate": couchurl,
"attr": "acdcServer"
},
"ACDCDatabase": {
"default": "acdcserver",
"validate": identifier,
"attr": "acdcDatabase"
},
"CollectionName": {
"default": None,
"null": True
},
"IgnoredOutputModules": {
"default": [],
"type": makeList
},
"SiteWhitelist": {
"default": [],
"type": makeList,
"validate": lambda x: all([cmsname(y) for y in x])
},
# it can be Chained or MC requests, so lets make it optional
"InputDataset": {
"optional": True,
"validate": dataset,
"null": True
}
}
StdBase.setDefaultArgumentsProperty(specArgs)
return specArgs
def getWorkloadAssignArgs():
baseArgs = StdBase.getWorkloadAssignArgs()
specArgs = {
"Override": {
"default": {
"eos-lfn-prefix":
"root://eoscms.cern.ch//eos/cms/store/logs/prod/recent/Repack"
},
"type": dict
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {"RequestType" : {"default" : "StoreResults", "optional" : True,
"attr" : "requestType"},
"InputDataset" : {"default" : None,
"type" : str, "optional" : False,
"validate" : dataset, "attr" : "inputDataset",
"null" : False},
"GlobalTag" : {"default" : "GT_SR_V1:All", "type" : str,
"optional" : False, "validate" : None,
"attr" : "globalTag", "null" : False},
"CmsPath" : {"default" : "/tmp", "type" : str,
"optional" : False, "validate" : None,
"attr" : "cmsPath", "null" : False},
"DataTier" : {"default" : "USER", "type" : str,
"optional" : True, "validate" : None,
"attr" : "dataTier", "null" : False},
"UnmergedLFNBase" : {"default" : "/store/unmerged", "type" : str,
"optional" : True, "validate" : None,
"attr" : "unmergedLFNBase", "null" : False},
"MergedLFNBase" : {"default" : "/store/results", "type" : str,
"optional" : True, "validate" : None,
"attr" : "mergedLFNBase", "null" : False},
"MinMergeSize" : {"default" : 2 * 1024 * 1024 * 1024, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "minMergeSize", "null" : False},
"MaxMergeSize" : {"default" : 4 * 1024 * 1024 * 1024, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "maxMergeSize", "null" : False},
"MaxMergeEvents" : {"default" : 100000, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "maxMergeEvents", "null" : False},
"BlockBlacklist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([block(y) for y in x]),
"attr" : "blockBlacklist", "null" : False},
"BlockWhitelist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([block(y) for y in x]),
"attr" : "blockWhitelist", "null" : False},
"RunBlacklist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]),
"attr" : "runBlacklist", "null" : False},
"RunWhitelist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]),
"attr" : "runWhitelist", "null" : False}}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {"RequestType": {"default": "MonteCarlo", "optional": True,
"attr": "requestType"},
"PrimaryDataset": {"default": "BlackHoleTest", "type": str,
"optional": False, "validate": primdataset,
"attr": "inputPrimaryDataset", "null": False},
"Seeding": {"default": "AutomaticSeeding", "type": str,
"optional": True, "validate": lambda x: x in ["ReproducibleSeeding", "AutomaticSeeding"],
"attr": "seeding", "null": False},
"GlobalTag": {"default": "GT_MC_V1:All", "type": str,
"optional": False, "validate": None,
"attr": "globalTag", "null": False},
"FilterEfficiency": {"default": 1.0, "type": float,
"optional": True, "validate": lambda x: x > 0.0,
"attr": "filterEfficiency", "null": False},
"RequestNumEvents": {"default": 1000, "type": int,
"optional": False, "validate": lambda x: x > 0,
"attr": "requestNumEvents", "null": False},
"FirstEvent": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstEvent", "null": False},
"FirstLumi": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstLumi", "null": False},
"MCPileup": {"default": None, "type": str,
"optional": True, "validate": dataset,
"attr": "mcPileup", "null": False},
"DataPileup": {"default": None, "type": str,
"optional": True, "validate": dataset,
"attr": "dataPileup", "null": False},
"DeterministicPileup": {"default": False, "type": strToBool,
"optional": True, "validate": None,
"attr": "deterministicPileup", "null": False},
"EventsPerJob": {"default": None, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "eventsPerJob", "null": True},
"EventsPerLumi": {"default": None, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "eventsPerLumi", "null": True},
"LheInputFiles": {"default": False, "type": strToBool,
"optional": True, "validate": None,
"attr": "lheInputFiles", "null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {"RequestType": {"default": "MonteCarlo", "optional": True,
"attr": "requestType"},
"PrimaryDataset": {"default": "BlackHoleTest", "type": str,
"optional": False, "validate": primdataset,
"attr": "inputPrimaryDataset", "null": False},
"Seeding": {"default": "AutomaticSeeding", "type": str,
"optional": True, "validate": lambda x: x in ["ReproducibleSeeding", "AutomaticSeeding"],
"attr": "seeding", "null": False},
"GlobalTag": {"default": "GT_MC_V1:All", "type": str,
"optional": False, "validate": None,
"attr": "globalTag", "null": False},
"FilterEfficiency": {"default": 1.0, "type": float,
"optional": True, "validate": lambda x: x > 0.0,
"attr": "filterEfficiency", "null": False},
"RequestNumEvents": {"default": 1000, "type": int,
"optional": False, "validate": lambda x: x > 0,
"attr": "requestNumEvents", "null": False},
"FirstEvent": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstEvent", "null": False},
"FirstLumi": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstLumi", "null": False},
"MCPileup": {"default": None, "type": str,
"optional": True, "validate": dataset,
"attr": "mcPileup", "null": True},
"DataPileup": {"default": None, "type": str,
"optional": True, "validate": dataset,
"attr": "dataPileup", "null": True},
"DeterministicPileup": {"default": False, "type": strToBool,
"optional": True, "validate": None,
"attr": "deterministicPileup", "null": False},
"EventsPerJob": {"default": None, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "eventsPerJob", "null": True},
"EventsPerLumi": {"default": None, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "eventsPerLumi", "null": True},
"LheInputFiles": {"default": False, "type": strToBool,
"optional": True, "validate": None,
"attr": "lheInputFiles", "null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {"RequestType" : {"default" : "StoreResults", "optional" : True,
"attr" : "requestType"},
"InputDataset" : {"default" : None,
"type" : str, "optional" : False,
"validate" : dataset, "attr" : "inputDataset",
"null" : False},
"GlobalTag" : {"default" : "GT_SR_V1:All", "type" : str,
"optional" : False, "validate" : None,
"attr" : "globalTag", "null" : False},
"CmsPath" : {"default" : "/tmp", "type" : str,
"optional" : False, "validate" : None,
"attr" : "cmsPath", "null" : False},
"DataTier" : {"default" : "USER", "type" : str,
"optional" : True, "validate" : None,
"attr" : "dataTier", "null" : False},
"UnmergedLFNBase" : {"default" : "/store/unmerged", "type" : str,
"optional" : True, "validate" : None,
"attr" : "unmergedLFNBase", "null" : False},
"MergedLFNBase" : {"default" : "/store/results", "type" : str,
"optional" : True, "validate" : None,
"attr" : "mergedLFNBase", "null" : False},
"MinMergeSize" : {"default" : 2 * 1024 * 1024 * 1024, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "minMergeSize", "null" : False},
"MaxMergeSize" : {"default" : 4 * 1024 * 1024 * 1024, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "maxMergeSize", "null" : False},
"MaxMergeEvents" : {"default" : 100000, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "maxMergeEvents", "null" : False},
"BlockBlacklist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([block(y) for y in x]),
"attr" : "blockBlacklist", "null" : False},
"BlockWhitelist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([block(y) for y in x]),
"attr" : "blockWhitelist", "null" : False},
"RunBlacklist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]),
"attr" : "runBlacklist", "null" : False},
"RunWhitelist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]),
"attr" : "runWhitelist", "null" : False}}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "StepChain", "optional": False},
"Step1": {"default": {}, "optional": False, "type": dict},
# ConfigCacheID is not used in the main dict for StepChain
"ConfigCacheID": {"optional": True, "null": True},
"DeterministicPileup": {"default": False, "type": strToBool, "optional": True, "null": False},
"PrimaryDataset": {"null": True, "validate": primdataset},
"StepChain": {"default": 1, "type": int, "null": False,
"optional": False, "validate": lambda x: x > 0},
"FirstEvent": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False},
"FirstLumi": {"default": 1, "type": int, "validate": lambda x: x > 0,
"null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {"InputDataset" : {"default" : "/MinimumBias/ComissioningHI-v1/RAW", "type" : str,
"optional" : False, "validate" : dataset,
"attr" : "inputDataset", "null" : False},
"GlobalTag" : {"default" : "GT_DP_V1", "type" : str,
"optional" : False, "validate" : None,
"attr" : "globalTag", "null" : False},
"OpenRunningTimeout" : {"default" : 0, "type" : int,
"optional" : True, "validate" : lambda x : x >= 0,
"attr" : "openRunningTimeout", "null" : False},
"BlockBlacklist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([block(y) for y in x]),
"attr" : "blockBlacklist", "null" : False},
"BlockWhitelist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([block(y) for y in x]),
"attr" : "blockWhitelist", "null" : False},
"RunBlacklist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]),
"attr" : "runBlacklist", "null" : False},
"RunWhitelist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]),
"attr" : "runWhitelist", "null" : False},
"SplittingAlgo" : {"default" : "EventAwareLumiBased", "type" : str,
"optional" : True, "validate" : lambda x: x in ["EventBased", "LumiBased",
"EventAwareLumiBased", "FileBased"],
"attr" : "procJobSplitAlgo", "null" : False},
"EventsPerJob" : {"default" : None, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "eventsPerJob", "null" : True},
"LumisPerJob" : {"default" : 8, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "lumisPerJob", "null" : False},
"FilesPerJob" : {"default" : 1, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "filesPerJob", "null" : False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
specArgs = {"RequestType": {"default": "Resubmission"},
"OriginalRequestType": {"null": False},
"OriginalRequestName": {"null": False},
"InitialTaskPath": {"optional": False,
"validate": lambda x: len(x.split('/')) > 2},
"ACDCServer": {"default": "https://cmsweb.cern.ch/couchdb", "validate": couchurl,
"attr": "acdcServer"},
"ACDCDatabase": {"default": "acdcserver", "validate": identifier,
"attr": "acdcDatabase"},
"CollectionName": {"default": None, "null": True},
"IgnoredOutputModules": {"default": [], "type": makeList},
"SiteWhitelist": {"default": [], "type": makeList,
"validate": lambda x: all([cmsname(y) for y in x])},
# it can be Chained or MC requests, so lets make it optional
"InputDataset": {"optional": True, "validate": dataset, "null": True}}
StdBase.setDefaultArgumentsProperty(specArgs)
return specArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "TaskChain", "optional": False},
"Task1": {"default": {}, "optional": False, "type": dict},
# ConfigCacheID is not used in the main dict for TaskChain
"ConfigCacheID": {"optional": True, "null": True},
"IgnoredOutputModules": {"default": [], "type": makeList, "null": False},
"TaskChain": {"default": 1, "type": int,
"optional": False, "validate": lambda x: x > 0,
"attr": "taskChain", "null": False},
"FirstEvent": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstEvent", "null": False},
"FirstLumi": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "firstLumi", "null": False}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "Repack"},
"ConfigCacheID": {"optional": True, "null": True},
"Scenario": {"default": "fake", "attr": "procScenario"},
"GlobalTag": {"default": "fake"},
"ProcessingString": {"default": "", "validate": procstringT0},
"Outputs": {"type": makeList, "optional": False},
"MaxSizeSingleLumi": {"type": int, "optional": False},
"MaxSizeMultiLumi": {"type": int, "optional": False},
"MaxInputEvents": {"type": int, "optional": False},
"MaxInputFiles": {"type": int, "optional": False},
"MaxLatency": {"type": int, "optional": False},
"MinInputSize": {"type": int, "optional": False},
"MaxInputSize": {"type": int, "optional": False},
"MaxEdmSize": {"type": int, "optional": False},
"MaxOverSize": {"type": int, "optional": False},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
specArgs = {"RequestType" : {"default" : "StepChain", "optional" : False},
"GlobalTag" : {"type" : str, "optional" : False},
"CouchURL" : {"type" : str, "optional" : False, "validate" : couchurl},
"CouchDBName" : {"type" : str, "optional" : False,
"validate" : identifier},
"ConfigCacheUrl" : {"type" : str, "optional" : True, "null" : True},
"StepChain" : {"default" : 1, "type" : int,
"optional" : False, "validate" : lambda x: x > 0,
"attr" : "stepChain", "null" : False},
"FirstEvent" : {"default" : 1, "type" : int,
"optional" : True, "validate" : lambda x: x > 0,
"attr" : "firstEvent", "null" : False},
"FirstLumi" : {"default" : 1, "type" : int,
"optional" : True, "validate" : lambda x: x > 0,
"attr" : "firstLumi", "null" : False},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getChainCreateArgs(firstTask=False, generator=False):
"""
_getChainCreateArgs_
This represents the authoritative list of request arguments that are
allowed in each chain (Step/Task) of chained request, during request creation.
Additional especific arguments must be defined inside each spec class.
For more information on how these arguments are built, please have a look
at the docstring for getWorkloadCreateArgs.
"""
baseArgs = StdBase.getChainCreateArgs(firstTask, generator)
arguments = {
'InputStep': {'default': None, 'null': False, 'optional': firstTask, 'type': str},
'StepName': {'null': False, 'optional': False},
'PrimaryDataset': {'default': None, 'optional': True,
'validate': primdataset, 'null': False}
}
baseArgs.update(arguments)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadAssignArgs():
baseArgs = StdBase.getWorkloadAssignArgs()
specArgs = {
"ChainParentageMap": {
"default": {},
"type": dict
},
### Override StdBase assignment parameter definition
"Memory": {
"type": float,
"validate": checkMemCore
},
"Multicore": {
"type": int,
"validate": checkMemCore
},
"EventStreams": {
"type": int,
"validate": checkEventStreams
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getChainCreateArgs(firstTask=False, generator=False):
"""
_getChainCreateArgs_
Each task dictionary specifies its own set of arguments
that need to be validated as well, most of them are already
defined in StdBase.getWorkloadCreateArgs and those do not appear here
since they are all optional. Here only new arguments are listed.
"""
baseArgs = StdBase.getChainCreateArgs(firstTask, generator)
arguments = {
"TaskName": {"optional": False, "null": False},
"InputTask": {"default": None, "optional": firstTask, "null": False},
"TransientOutputModules": {"default": [], "type": makeList, "optional": True, "null": False},
"DeterministicPileup": {"default": False, "type": strToBool, "optional": True, "null": False},
"GlobalTag": {"type": str, "optional": True},
"TimePerEvent": {"type": float, "optional": True, "validate": lambda x: x > 0},
"SizePerEvent": {"type": float, "optional": True, "validate": lambda x: x > 0},
'PrimaryDataset': {'default': None, 'optional': not generator, 'validate': primdataset,
'null': False},
}
baseArgs.update(arguments)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
specArgs = {"RequestType" : {"default" : "PromptReco", "optional" : True,
"attr" : "requestType"},
"Scenario" : {"default" : None, "type" : str,
"optional" : False, "validate" : None,
"attr" : "procScenario", "null" : False},
"GlobalTag" : {"default" : None, "type" : str,
"optional" : False, "validate" : None,
"attr" : "globalTag", "null" : False},
"ProcessingString": {"default": "", "validate": procstringT0},
"WriteTiers" : {"default" : ["RECO", "AOD", "DQM", "ALCARECO"],
"type" : makeList, "optional" : False,
"validate" : None,
"attr" : "writeTiers", "null" : False},
"AlcaSkims" : {"default" : ["TkAlCosmics0T","MuAlGlobalCosmics","HcalCalHOCosmics"],
"type" : makeList, "optional" : False,
"validate" : None,
"attr" : "alcaSkims", "null" : False},
"InputDataset" : {"default" : "/Cosmics/Run2012A-v1/RAW", "type" : str,
"optional" : False, "validate" : dataset,
"attr" : "inputDataset", "null" : False},
"PhysicsSkims" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : None,
"attr" : "physicsSkims", "null" : False},
"InitCommand" : {"default" : None, "type" : str,
"optional" : True, "validate" : None,
"attr" : "initCommand", "null" : True},
"EnvPath" : {"default" : None, "type" : str,
"optional" : True, "validate" : None,
"attr" : "envPath", "null" : True},
"BinPath" : {"default" : None, "type" : str,
"optional" : True, "validate" : None,
"attr" : "binPath", "null" : True},
"DoLogCollect" : {"default" : True, "type" : strToBool,
"optional" : True, "validate" : None,
"attr" : "doLogCollect", "null" : False},
"BlockBlacklist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([block(y) for y in x]),
"attr" : "blockBlacklist", "null" : False},
"BlockWhitelist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([block(y) for y in x]),
"attr" : "blockWhitelist", "null" : False},
"RunBlacklist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]),
"attr" : "runBlacklist", "null" : False},
"RunWhitelist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]),
"attr" : "runWhitelist", "null" : False},
"SplittingAlgo" : {"default" : "EventBased", "type" : str,
"optional" : True, "validate" : lambda x: x in ["EventBased", "LumiBased",
"EventAwareLumiBased", "FileBased"],
"attr" : "procJobSplitAlgo", "null" : False},
"EventsPerJob" : {"default" : 500, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "eventsPerJob", "null" : False},
"LumisPerJob" : {"default" : 8, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "lumisPerJob", "null" : False},
"FilesPerJob" : {"default" : 1, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "filesPerJob", "null" : False},
"SkimSplittingAlgo" : {"default" : "FileBased", "type" : str,
"optional" : True, "validate" : lambda x: x in ["EventBased", "LumiBased",
"EventAwareLumiBased", "FileBased"],
"attr" : "skimJobSplitAlgo", "null" : False},
"SkimEventsPerJob" : {"default" : 500, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "skimEventsPerJob", "null" : False},
"SkimLumisPerJob" : {"default" : 8, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "skimLumisPerJob", "null" : False},
"SkimFilesPerJob" : {"default" : 1, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "skimFilesPerJob", "null" : False},
"BlockCloseDelay" : {"default" : 86400, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "blockCloseDelay", "null" : False}
}
baseArgs.update(specArgs)
# add more optional arguments in case it is created using ReqMgr (not T0 case but should support both)
reqMgrArguments = {"CouchURL" : {"default" : "https://cmsweb.cern.ch/couchdb",
"validate" : couchurl},
"CouchDBName" : {"default" : "reqmgr_config_cache", "type" : str,
"validate" : identifier},
"ConfigCacheUrl" : {"default" :"https://cmsweb.cern.ch/couchdb", "validate" : couchurl},
"ConfigCacheID" : {"optional": True, "null": True},
"CouchWorkloadDBName" : {"default" : "reqmgr_workload_cache", "validate" : identifier},
}
baseArgs.update(reqMgrArguments)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "StepChain",
"optional": False
},
"Step1": {
"default": {},
"optional": False,
"type": dict
},
# ConfigCacheID is not used in the main dict for StepChain
"ConfigCacheID": {
"optional": True,
"null": True
},
"DeterministicPileup": {
"default": False,
"type": strToBool,
"optional": True,
"null": False
},
"PrimaryDataset": {
"null": True,
"validate": primdataset
},
"StepChain": {
"default": 1,
"type": int,
"null": False,
"optional": False,
"validate": lambda x: x > 0
},
"ChainParentageMap": {
"default": {},
"type": dict
},
"FirstEvent": {
"default": 1,
"type": int,
"validate": lambda x: x > 0,
"null": False
},
"FirstLumi": {
"default": 1,
"type": int,
"validate": lambda x: x > 0,
"null": False
},
"ParentageResolved": {
"default": False,
"type": strToBool,
"null": False
},
### Override StdBase parameter definition
"TimePerEvent": {
"default": 12.0,
"type": float,
"validate": lambda x: x > 0
},
"Memory": {
"default": 2300.0,
"type": float,
"validate": lambda x: x > 0
},
"Multicore": {
"default": 1,
"type": int,
"validate": checkMemCore
},
"EventStreams": {
"type": int,
"null": True,
"default": 0,
"validate": checkEventStreams
}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getTaskArguments(firstTask=False, generator=False):
"""
_getTaskArguments_
Each task dictionary specifies its own set of arguments
that need to be validated as well, most of them are already
defined in StdBase.getWorkloadArguments and those do not appear here
since they are all optional. Here only new arguments are listed.
"""
specArgs = {
"TaskName": {
"default": None,
"type": str,
"optional": False,
"validate": None,
"null": False
},
"ConfigCacheUrl": {
"default": "https://cmsweb.cern.ch/couchdb",
"type": str,
"optional": False,
"validate": None,
"attr": "configCacheUrl",
"null": False
},
"ConfigCacheID": {
"default": None,
"type": str,
"optional": False,
"validate": None,
"null": False
},
"KeepOutput": {
"default": True,
"type": strToBool,
"optional": True,
"validate": None,
"null": False
},
"TransientOutputModules": {
"default": [],
"type": makeList,
"optional": True,
"validate": None,
"null": False
},
"PrimaryDataset": {
"default": None,
"type": str,
"optional": not generator,
"validate": primdataset,
"null": False
},
"Seeding": {
"default":
"AutomaticSeeding",
"type":
str,
"optional":
True,
"validate":
lambda x: x in ["ReproducibleSeeding", "AutomaticSeeding"],
"null":
False
},
"RequestNumEvents": {
"default": 1000,
"type": int,
"optional": not generator,
"validate": lambda x: x > 0,
"null": False
},
"MCPileup": {
"default": None,
"type": str,
"optional": True,
"validate": dataset,
"null": False
},
"DataPileup": {
"default": None,
"type": str,
"optional": True,
"validate": dataset,
"null": False
},
"DeterministicPileup": {
"default": False,
"type": strToBool,
"optional": True,
"validate": None,
"attr": "deterministicPileup",
"null": False
},
"InputDataset": {
"default": None,
"type": str,
"optional": generator or not firstTask,
"validate": dataset,
"null": False
},
"InputTask": {
"default": None,
"type": str,
"optional": firstTask,
"validate": None,
"null": False
},
"InputFromOutputModule": {
"default": None,
"type": str,
"optional": firstTask,
"validate": None,
"null": False
},
"BlockBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"null": False
},
"BlockWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"null": False
},
"RunBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"null": False
},
"RunWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"null": False
},
"SplittingAlgo": {
"default":
"EventAwareLumiBased",
"type":
str,
"optional":
True,
"validate":
lambda x: x in [
"EventBased", "LumiBased", "EventAwareLumiBased",
"FileBased"
],
"null":
False
},
"EventsPerJob": {
"default": None,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"null": False
},
"LumisPerJob": {
"default": 8,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"null": False
},
"FilesPerJob": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"null": False
},
"EventsPerLumi": {
"default": None,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "eventsPerLumi",
"null": True
},
"FilterEfficiency": {
"default": 1.0,
"type": float,
"optional": True,
"validate": lambda x: x > 0.0,
"attr": "filterEfficiency",
"null": False
},
"LheInputFiles": {
"default": False,
"type": strToBool,
"optional": True,
"validate": None,
"attr": "lheInputFiles",
"null": False
},
"PrepID": {
"default": None,
"type": str,
"optional": True,
"validate": None,
"attr": "prepID",
"null": True
},
"Multicore": {
"default": None,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"null": False
},
}
StdBase.setDefaultArgumentsProperty(specArgs)
return specArgs
def getTaskArguments(firstTask=False, generator=False):
"""
_getTaskArguments_
Each task dictionary specifies its own set of arguments
that need to be validated as well, most of them are already
defined in StdBase.getWorkloadArguments and those do not appear here
since they are all optional. Here only new arguments are listed.
"""
specArgs = {"TaskName": {"default": None, "type": str,
"optional": False, "validate": None,
"null": False},
"ConfigCacheUrl": {"default": "https://cmsweb.cern.ch/couchdb", "type": str,
"optional": False, "validate": None,
"attr": "configCacheUrl", "null": False},
"ConfigCacheID": {"default": None, "type": str,
"optional": False, "validate": None,
"null": False},
"KeepOutput": {"default": True, "type": strToBool,
"optional": True, "validate": None,
"null": False},
"TransientOutputModules": {"default": [], "type": makeList,
"optional": True, "validate": None,
"null": False},
"PrimaryDataset": {"default": None, "type": str,
"optional": not generator, "validate": primdataset,
"null": False},
"Seeding": {"default": "AutomaticSeeding", "type": str,
"optional": True,
"validate": lambda x: x in ["ReproducibleSeeding", "AutomaticSeeding"],
"null": False},
"RequestNumEvents": {"default": 1000, "type": int,
"optional": not generator, "validate": lambda x: x > 0,
"null": False},
"MCPileup": {"default": None, "type": str,
"optional": True, "validate": dataset,
"null": False},
"DataPileup": {"default": None, "type": str,
"optional": True, "validate": dataset,
"null": False},
"DeterministicPileup": {"default": False, "type": strToBool,
"optional": True, "validate": None,
"attr": "deterministicPileup", "null": False},
"InputDataset": {"default": None, "type": str,
"optional": generator or not firstTask, "validate": dataset,
"null": False},
"InputTask": {"default": None, "type": str,
"optional": firstTask, "validate": None,
"null": False},
"InputFromOutputModule": {"default": None, "type": str,
"optional": firstTask, "validate": None,
"null": False},
"BlockBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"null": False},
"BlockWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([block(y) for y in x]),
"null": False},
"RunBlacklist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"null": False},
"RunWhitelist": {"default": [], "type": makeList,
"optional": True, "validate": lambda x: all([int(y) > 0 for y in x]),
"null": False},
"SplittingAlgo": {"default": "EventAwareLumiBased", "type": str,
"optional": True, "validate": lambda x: x in ["EventBased", "LumiBased",
"EventAwareLumiBased", "FileBased"],
"null": False},
"EventsPerJob": {"default": None, "type": int,
"optional": True, "validate": lambda x: x > 0,
"null": False},
"LumisPerJob": {"default": 8, "type": int,
"optional": True, "validate": lambda x: x > 0,
"null": False},
"FilesPerJob": {"default": 1, "type": int,
"optional": True, "validate": lambda x: x > 0,
"null": False},
"EventsPerLumi": {"default": None, "type": int,
"optional": True, "validate": lambda x: x > 0,
"attr": "eventsPerLumi", "null": True},
"FilterEfficiency": {"default": 1.0, "type": float,
"optional": True, "validate": lambda x: x > 0.0,
"attr": "filterEfficiency", "null": False},
"LheInputFiles": {"default": False, "type": strToBool,
"optional": True, "validate": None,
"attr": "lheInputFiles", "null": False},
"PrepID": {"default": None, "type": str,
"optional": True, "validate": None,
"attr": "prepID", "null": True},
"Multicore": {"default": None, "type": int,
"optional": True, "validate": lambda x: x > 0,
"null": False},
}
StdBase.setDefaultArgumentsProperty(specArgs)
return specArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
del baseArgs["AcquisitionEra"]
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {"RequestType" : {"default" : "Analysis", "optional" : True,
"attr" : "requestType"},
"userSandbox" : {"default" : "http://home.fnal.gov/~ewv/agent.tgz",
"type" : str,
"optional" : True, "validate" : None,
"attr" : "userSandbox", "null" : True},
"userFiles" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : None,
"attr" : "userFiles", "null" : False},
"OutputFiles" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : None,
"attr" : "outputFiles", "null" : False},
"Username" : {"default" : "mmascher", "type" : str,
"optional" : True, "validate" : None,
"attr" : "userName", "null" : False},
"SaveLogs" : {"default" : True, "type" : strToBool,
"optional" : True, "validate" : None,
"attr" : "saveLogs", "null" : False},
"CouchURL" : {"default" : "http://localhost:5984", "type" : str,
"optional" : False, "validate" : couchurl,
"attr" : "couchURL", "null" : False},
"CouchDBName" : {"default" : "analysis_reqmgr_config_cache", "type" : str,
"optional" : False, "validate" : identifier,
"attr" : "couchDBName", "null" : False},
"AnalysisConfigCacheDoc" : {"default" : None, "type" : str,
"optional" : False, "validate" : None,
"attr" : "configCacheID", "null" : False},
"ConfigCacheUrl" : {"default" : None, "type" : str,
"optional" : True, "validate" : None,
"attr" : "configCacheUrl", "null" : False},
"PublishDataName" : {"default" : None, "type" : str,
"optional" : True, "validate" : None,
"attr" : "acquisitionEra", "null" : False},
"GlobalTag" : {"default" : "GT_AN_V1:All", "type" : str,
"optional" : False, "validate" : None,
"attr" : "globalTag", "null" : False},
"InputDataset" : {"default" : "/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO",
"type" : str, "optional" : False,
"validate" : None, "attr" : "inputDataset",
"null" : False},
"OriginalRequestName" : {"default" : None, "type" : str,
"optional" : True, "validate" : None,
"attr" : "origRequest", "null" : False},
"BlockBlacklist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([block(y) for y in x]),
"attr" : "blockBlacklist", "null" : False},
"BlockWhitelist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([block(y) for y in x]),
"attr" : "blockWhitelist", "null" : False},
"RunBlacklist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]),
"attr" : "runBlacklist", "null" : False},
"RunWhitelist" : {"default" : [], "type" : makeList,
"optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]),
"attr" : "runWhitelist", "null" : False},
"asyncDest" : {"default" : "T1_US_FNAL_Buffer", "type" : str,
"optional" : True, "validate" : None,
"attr" : "asyncDest", "null" : False},
"ACDCUrl" : {"default" : "http://cmsweb-testbed.cern.ch/couchdb",
"type" : str,
"optional" : True, "validate" : couchurl,
"attr" : "ACDCURL", "null" : False},
"ACDCDBName" : {"default" : "acdcserver", "type" : str,
"optional" : True, "validate" : identifier,
"attr" : "ACDCDBName", "null" : False},
"Runs" : {"default" : None, "type" : makeList,
"optional" : True, "validate" : None,
"attr" : "Runs", "null" : False},
"Lumis" : {"default" : None, "type" : makeList,
"optional" : True, "validate" : None,
"attr" : "Lumis", "null" : False},
"Submission" : {"default" : 1, "type" : int,
"optional" : True, "validate" : None,
"attr" : "Submission", "null" : False},
"JobSplitAlgo" : {"default" : "EventBased", "type" : str,
"optional" : True, "validate" : None,
"attr" : "analysisJobSplitAlgo", "null" : False},
"EventsPerJob" : {"default" : 1000, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "eventsPerJob", "null" : False},
"LumisPerJob" : {"default" : 8, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "lumisPerJob", "null" : False}, }
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
del baseArgs["AcquisitionEra"]
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {
"RequestType": {
"default": "Analysis",
"optional": True,
"attr": "requestType"
},
"userSandbox": {
"default": "http://home.fnal.gov/~ewv/agent.tgz",
"type": str,
"optional": True,
"validate": None,
"attr": "userSandbox",
"null": True
},
"userFiles": {
"default": [],
"type": makeList,
"optional": True,
"validate": None,
"attr": "userFiles",
"null": False
},
"OutputFiles": {
"default": [],
"type": makeList,
"optional": True,
"validate": None,
"attr": "outputFiles",
"null": False
},
"Username": {
"default": "mmascher",
"type": str,
"optional": True,
"validate": None,
"attr": "userName",
"null": False
},
"SaveLogs": {
"default": True,
"type": strToBool,
"optional": True,
"validate": None,
"attr": "saveLogs",
"null": False
},
"CouchURL": {
"default": "http://localhost:5984",
"type": str,
"optional": False,
"validate": couchurl,
"attr": "couchURL",
"null": False
},
"CouchDBName": {
"default": "analysis_reqmgr_config_cache",
"type": str,
"optional": False,
"validate": identifier,
"attr": "couchDBName",
"null": False
},
"AnalysisConfigCacheDoc": {
"default": None,
"type": str,
"optional": False,
"validate": None,
"attr": "configCacheID",
"null": False
},
"ConfigCacheUrl": {
"default": None,
"type": str,
"optional": True,
"validate": None,
"attr": "configCacheUrl",
"null": False
},
"PublishDataName": {
"default": None,
"type": str,
"optional": True,
"validate": None,
"attr": "acquisitionEra",
"null": False
},
"GlobalTag": {
"default": "GT_AN_V1:All",
"type": str,
"optional": False,
"validate": None,
"attr": "globalTag",
"null": False
},
"InputDataset": {
"default":
"/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO",
"type": str,
"optional": False,
"validate": None,
"attr": "inputDataset",
"null": False
},
"OriginalRequestName": {
"default": None,
"type": str,
"optional": True,
"validate": None,
"attr": "origRequest",
"null": False
},
"BlockBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"attr": "blockBlacklist",
"null": False
},
"BlockWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"attr": "blockWhitelist",
"null": False
},
"RunBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runBlacklist",
"null": False
},
"RunWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runWhitelist",
"null": False
},
"asyncDest": {
"default": "T1_US_FNAL_Buffer",
"type": str,
"optional": True,
"validate": None,
"attr": "asyncDest",
"null": False
},
"ACDCUrl": {
"default": "http://cmsweb-testbed.cern.ch/couchdb",
"type": str,
"optional": True,
"validate": couchurl,
"attr": "ACDCURL",
"null": False
},
"ACDCDBName": {
"default": "acdcserver",
"type": str,
"optional": True,
"validate": identifier,
"attr": "ACDCDBName",
"null": False
},
"Runs": {
"default": None,
"type": makeList,
"optional": True,
"validate": None,
"attr": "Runs",
"null": False
},
"Lumis": {
"default": None,
"type": makeList,
"optional": True,
"validate": None,
"attr": "Lumis",
"null": False
},
"Submission": {
"default": 1,
"type": int,
"optional": True,
"validate": None,
"attr": "Submission",
"null": False
},
"JobSplitAlgo": {
"default": "EventBased",
"type": str,
"optional": True,
"validate": None,
"attr": "analysisJobSplitAlgo",
"null": False
},
"EventsPerJob": {
"default": 1000,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "eventsPerJob",
"null": False
},
"LumisPerJob": {
"default": 8,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "lumisPerJob",
"null": False
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "TaskChain",
"optional": False
},
"Task1": {
"default": {},
"optional": False,
"type": dict
},
# ConfigCacheID is not used in the main dict for TaskChain
"ConfigCacheID": {
"optional": True,
"null": True
},
"IgnoredOutputModules": {
"default": [],
"type": makeList,
"null": False
},
"TaskChain": {
"default": 1,
"type": int,
"optional": False,
"validate": lambda x: x > 0,
"attr": "taskChain",
"null": False
},
"ChainParentageMap": {
"default": {},
"type": dict
},
"FirstEvent": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "firstEvent",
"null": False
},
"FirstLumi": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "firstLumi",
"null": False
},
### Override StdBase parameter definition
"TimePerEvent": {
"default": 12.0,
"type": float,
"validate": checkTimePerEvent
},
"Memory": {
"default": 2300.0,
"type": float,
"validate": checkMemCore
},
"Multicore": {
"default": 1,
"type": int,
"validate": checkMemCore
},
"EventStreams": {
"type": int,
"null": True,
"default": 0,
"validate": checkEventStreams
},
# no need for workload-level defaults, if task-level default is provided
"RequiresGPU": {
"default": None,
"null": True,
"validate": lambda x: x in
("forbidden", "optional", "required")
},
"GPUParams": {
"default": json.dumps(None),
"validate": gpuParameters
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "Express"
},
"ConfigCacheID": {
"optional": True,
"null": True
},
"Scenario": {
"optional": False,
"attr": "procScenario"
},
"RecoCMSSWVersion": {
"validate": lambda x: x in releases(),
"optional": False,
"attr": "recoFrameworkVersion"
},
"RecoScramArch": {
"validate": lambda x: all([y in architectures() for y in x]),
"optional": False,
"type": makeNonEmptyList
},
"GlobalTag": {
"optional": False
},
"GlobalTagTransaction": {
"optional": False
},
"ProcessingString": {
"default": "",
"validate": procstringT0
},
"StreamName": {
"optional": False
},
"SpecialDataset": {
"optional": False
},
"AlcaHarvestTimeout": {
"type": int,
"optional": False
},
"AlcaHarvestDir": {
"optional": False,
"null": True
},
"AlcaSkims": {
"type": makeList,
"optional": False
},
"DQMSequences": {
"type": makeList,
"attr": "dqmSequences",
"optional": False
},
"Outputs": {
"type": makeList,
"optional": False
},
"MaxInputRate": {
"type": int,
"optional": False
},
"MaxInputEvents": {
"type": int,
"optional": False
},
"MaxInputSize": {
"type": int,
"optional": False
},
"MaxInputFiles": {
"type": int,
"optional": False
},
"MaxLatency": {
"type": int,
"optional": False
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
specArgs = {
"RequestType": {
"default": "PromptReco",
"optional": True,
"attr": "requestType"
},
"Scenario": {
"default": None,
"type": str,
"optional": False,
"validate": None,
"attr": "procScenario",
"null": False
},
"GlobalTag": {
"default": None,
"type": str,
"optional": False,
"validate": None,
"attr": "globalTag",
"null": False
},
"ProcessingString": {
"default": "",
"validate": procstringT0
},
"WriteTiers": {
"default": ["RECO", "AOD", "DQM", "ALCARECO"],
"type": makeList,
"optional": False,
"validate": None,
"attr": "writeTiers",
"null": False
},
"AlcaSkims": {
"default":
["TkAlCosmics0T", "MuAlGlobalCosmics", "HcalCalHOCosmics"],
"type":
makeList,
"optional":
False,
"validate":
None,
"attr":
"alcaSkims",
"null":
False
},
"InputDataset": {
"default": "/Cosmics/Run2012A-v1/RAW",
"type": str,
"optional": False,
"validate": dataset,
"attr": "inputDataset",
"null": False
},
"PhysicsSkims": {
"default": [],
"type": makeList,
"optional": True,
"validate": None,
"attr": "physicsSkims",
"null": False
},
"InitCommand": {
"default": None,
"type": str,
"optional": True,
"validate": None,
"attr": "initCommand",
"null": True
},
"EnvPath": {
"default": None,
"type": str,
"optional": True,
"validate": None,
"attr": "envPath",
"null": True
},
"BinPath": {
"default": None,
"type": str,
"optional": True,
"validate": None,
"attr": "binPath",
"null": True
},
"DoLogCollect": {
"default": True,
"type": strToBool,
"optional": True,
"validate": None,
"attr": "doLogCollect",
"null": False
},
"BlockBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"attr": "blockBlacklist",
"null": False
},
"BlockWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"attr": "blockWhitelist",
"null": False
},
"RunBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runBlacklist",
"null": False
},
"RunWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runWhitelist",
"null": False
},
"SplittingAlgo": {
"default":
"EventBased",
"type":
str,
"optional":
True,
"validate":
lambda x: x in [
"EventBased", "LumiBased", "EventAwareLumiBased",
"FileBased"
],
"attr":
"procJobSplitAlgo",
"null":
False
},
"EventsPerJob": {
"default": 500,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "eventsPerJob",
"null": False
},
"LumisPerJob": {
"default": 8,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "lumisPerJob",
"null": False
},
"FilesPerJob": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "filesPerJob",
"null": False
},
"SkimSplittingAlgo": {
"default":
"FileBased",
"type":
str,
"optional":
True,
"validate":
lambda x: x in [
"EventBased", "LumiBased", "EventAwareLumiBased",
"FileBased"
],
"attr":
"skimJobSplitAlgo",
"null":
False
},
"SkimEventsPerJob": {
"default": 500,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "skimEventsPerJob",
"null": False
},
"SkimLumisPerJob": {
"default": 8,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "skimLumisPerJob",
"null": False
},
"SkimFilesPerJob": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "skimFilesPerJob",
"null": False
},
"BlockCloseDelay": {
"default": 86400,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "blockCloseDelay",
"null": False
}
}
baseArgs.update(specArgs)
# add more optional arguments in case it is created using ReqMgr (not T0 case but should support both)
reqMgrArguments = {
"CouchURL": {
"default": "https://cmsweb.cern.ch/couchdb",
"validate": couchurl
},
"CouchDBName": {
"default": "reqmgr_config_cache",
"type": str,
"validate": identifier
},
"ConfigCacheUrl": {
"default": "https://cmsweb.cern.ch/couchdb",
"validate": couchurl
},
"ConfigCacheID": {
"optional": True,
"null": True
},
"CouchWorkloadDBName": {
"default": "reqmgr_workload_cache",
"validate": identifier
},
}
baseArgs.update(reqMgrArguments)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadAssignArgs():
baseArgs = StdBase.getWorkloadAssignArgs()
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadArguments():
baseArgs = StdBase.getWorkloadArguments()
reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr()
baseArgs.update(reqMgrArgs)
specArgs = {
"InputDataset": {
"default": "/MinimumBias/ComissioningHI-v1/RAW",
"type": str,
"optional": False,
"validate": dataset,
"attr": "inputDataset",
"null": False
},
"GlobalTag": {
"default": "GT_DP_V1",
"type": str,
"optional": False,
"validate": None,
"attr": "globalTag",
"null": False
},
"OpenRunningTimeout": {
"default": 0,
"type": int,
"optional": True,
"validate": lambda x: x >= 0,
"attr": "openRunningTimeout",
"null": False
},
"BlockBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"attr": "blockBlacklist",
"null": False
},
"BlockWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"attr": "blockWhitelist",
"null": False
},
"RunBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runBlacklist",
"null": False
},
"RunWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runWhitelist",
"null": False
},
"SplittingAlgo": {
"default":
"EventAwareLumiBased",
"type":
str,
"optional":
True,
"validate":
lambda x: x in [
"EventBased", "LumiBased", "EventAwareLumiBased",
"FileBased"
],
"attr":
"procJobSplitAlgo",
"null":
False
},
"EventsPerJob": {
"default": None,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "eventsPerJob",
"null": True
},
"LumisPerJob": {
"default": 8,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "lumisPerJob",
"null": False
},
"FilesPerJob": {
"default": 1,
"type": int,
"optional": True,
"validate": lambda x: x > 0,
"attr": "filesPerJob",
"null": False
}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "Repack"
},
"ConfigCacheID": {
"optional": True,
"null": True
},
"Scenario": {
"default": "fake",
"attr": "procScenario"
},
"GlobalTag": {
"default": "fake"
},
"ProcessingString": {
"default": "",
"validate": procstringT0
},
"Outputs": {
"type": makeList,
"optional": False
},
"MaxSizeSingleLumi": {
"type": int,
"optional": False
},
"MaxSizeMultiLumi": {
"type": int,
"optional": False
},
"MaxInputEvents": {
"type": int,
"optional": False
},
"MaxInputFiles": {
"type": int,
"optional": False
},
"MaxLatency": {
"type": int,
"optional": False
},
"MinInputSize": {
"type": int,
"optional": False
},
"MaxInputSize": {
"type": int,
"optional": False
},
"MaxEdmSize": {
"type": int,
"optional": False
},
"MaxOverSize": {
"type": int,
"optional": False
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "StoreResults",
"optional": False
},
"InputDataset": {
"optional": False,
"validate": dataset,
"null": False
},
"CmsPath": {
"default": "/tmp",
"type": str,
"optional": False,
"validate": None,
"attr": "cmsPath",
"null": False
},
"DataTier": {
"default": "USER",
"type": str,
"optional": True,
"validate": None,
"attr": "dataTier",
"null": False
},
"MergedLFNBase": {
"default": "/store/results",
"type": str,
"optional": True,
"validate": None,
"attr": "mergedLFNBase",
"null": False
},
"BlockBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"attr": "blockBlacklist",
"null": False
},
"BlockWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([block(y) for y in x]),
"attr": "blockWhitelist",
"null": False
},
"RunBlacklist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runBlacklist",
"null": False
},
"RunWhitelist": {
"default": [],
"type": makeList,
"optional": True,
"validate": lambda x: all([int(y) > 0 for y in x]),
"attr": "runWhitelist",
"null": False
}
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
