def getWorkloadCreateArgs(): baseArgs = StdBase.getWorkloadCreateArgs() specArgs = {"RequestType": {"default": "StoreResults", "optional": False}, "InputDataset": {"optional": False, "validate": dataset, "null": False}, "ConfigCacheID": {"optional": True, "null": True}, "DataTier": {"default": "USER", "type": str, "optional": True, "validate": None, "attr": "dataTier", "null": False}, "PhysicsGroup": {"default": "", "optional": False, "null": False, "validate": physicsgroup}, "MergedLFNBase": {"default": "/store/results", "type": str, "optional": True, "validate": None, "attr": "mergedLFNBase", "null": False}, "BlockBlacklist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockBlacklist", "null": False}, "BlockWhitelist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockWhitelist", "null": False}, "RunBlacklist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runBlacklist", "null": False}, "RunWhitelist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runWhitelist", "null": False}} baseArgs.update(specArgs) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr() baseArgs.update(reqMgrArgs) specArgs = {"RequestType": {"default": "DQMHarvest"}, "InputDataset": {"default": None, "optional": False, "validate": dataset}, "UnmergedLFNBase": {"default": "/store/unmerged"}, "MergedLFNBase": {"default": "/store/data"}, "MinMergeSize": {"default": 2 * 1024 * 1024 * 1024, "type": int, "validate": lambda x: x > 0}, "MaxMergeSize": {"default": 4 * 1024 * 1024 * 1024, "type": int, "validate": lambda x: x > 0}, "MaxMergeEvents": {"default": 100000, "type": int, "validate": lambda x: x > 0}, "BlockBlacklist": {"default": [], "type": makeList, "validate": lambda x: all([block(y) for y in x])}, "BlockWhitelist": {"default": [], "type": makeList, "validate": lambda x: all([block(y) for y in x])}, "RunBlacklist": {"default": [], "type": makeList, "validate": lambda x: all([int(y) > 0 for y in x])}, "RunWhitelist": {"default": [], "type": makeList, "validate": lambda x: all([int(y) > 0 for y in x])} } baseArgs.update(specArgs) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr() baseArgs.update(reqMgrArgs) specArgs = {"RequestType": {"default": "DQMHarvest"}, "InputDataset": {"default": None, "optional": False, "validate": dataset}, "ConfigCacheID": {"optional": True, "validate": None, "null": True}, "UnmergedLFNBase": {"default": "/store/unmerged"}, "MergedLFNBase": {"default": "/store/data"}, "MinMergeSize": {"default": 2 * 1024 * 1024 * 1024, "type": int, "validate": lambda x: x > 0}, "MaxMergeSize": {"default": 4 * 1024 * 1024 * 1024, "type": int, "validate": lambda x: x > 0}, "MaxMergeEvents": {"default": 100000, "type": int, "validate": lambda x: x > 0}, "BlockBlacklist": {"default": [], "type": makeList, "validate": lambda x: all([block(y) for y in x])}, "BlockWhitelist": {"default": [], "type": makeList, "validate": lambda x: all([block(y) for y in x])}, "RunBlacklist": {"default": [], "type": makeList, "validate": lambda x: all([int(y) > 0 for y in x])}, "RunWhitelist": {"default": [], "type": makeList, "validate": lambda x: all([int(y) > 0 for y in x])} } baseArgs.update(specArgs) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadCreateArgs(): baseArgs = StdBase.getWorkloadCreateArgs() specArgs = {"InputDataset": {"optional": False, "validate": dataset, "null": False}, "Scenario": {"optional": True, "null": True, "attr": "procScenario"}, "PrimaryDataset": {"optional": True, "validate": primdataset, "attr": "inputPrimaryDataset", "null": True}, "RunBlacklist": {"default": [], "type": makeList, "null": False, "validate": lambda x: all([int(y) > 0 for y in x])}, "RunWhitelist": {"default": [], "type": makeList, "null": False, "validate": lambda x: all([int(y) > 0 for y in x])}, "BlockBlacklist": {"default": [], "type": makeList, "validate": lambda x: all([block(y) for y in x])}, "BlockWhitelist": {"default": [], "type": makeList, "validate": lambda x: all([block(y) for y in x])}, "SplittingAlgo": {"default": "EventAwareLumiBased", "null": False, "validate": lambda x: x in ["EventBased", "LumiBased", "EventAwareLumiBased", "FileBased"], "attr": "procJobSplitAlgo"}, "EventsPerJob": {"type": int, "validate": lambda x: x > 0, "null": True}, "LumisPerJob": {"default": 8, "type": int, "null": False, "validate": lambda x: x > 0}, "FilesPerJob": {"default": 1, "type": int, "null": False, "validate": lambda x: x > 0} } baseArgs.update(specArgs) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadCreateArgs(): baseArgs = StdBase.getWorkloadCreateArgs() specArgs = {"RequestType": {"default": "StoreResults", "optional": False}, "InputDataset": {"optional": False, "validate": dataset, "null": False}, "ConfigCacheID": {"optional": True, "null": True}, "DataTier": {"default": "USER", "type": str, "optional": True, "validate": None, "attr": "dataTier", "null": False}, "PhysicsGroup": {"default": "", "optional": False, "null": False, "validate": physicsgroup}, "MergedLFNBase": {"default": "/store/results", "type": str, "optional": True, "validate": None, "attr": "mergedLFNBase", "null": False}, # site whitelist shouldn't be allowed, but let's make an exception for StoreResults "SiteWhitelist": {"default": [], "type": makeNonEmptyList, "assign_optional": False, "validate": lambda x: all([cmsname(y) for y in x])}, "BlockBlacklist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockBlacklist", "null": False}, "BlockWhitelist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockWhitelist", "null": False}, "RunBlacklist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runBlacklist", "null": False}, "RunWhitelist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runWhitelist", "null": False}} baseArgs.update(specArgs) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr() baseArgs.update(reqMgrArgs) specArgs = {"InputDataset" : {"default" : "/MinimumBias/ComissioningHI-v1/RAW", "type" : str, "optional" : False, "validate" : dataset, "attr" : "inputDataset", "null" : False}, "GlobalTag" : {"default" : "GT_DP_V1", "type" : str, "optional" : False, "validate" : None, "attr" : "globalTag", "null" : False}, "OpenRunningTimeout" : {"default" : 0, "type" : int, "optional" : True, "validate" : lambda x : x >= 0, "attr" : "openRunningTimeout", "null" : False}, "BlockBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockBlacklist", "null" : False}, "BlockWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockWhitelist", "null" : False}, "RunBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runBlacklist", "null" : False}, "RunWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runWhitelist", "null" : False}, "SplittingAlgo" : {"default" : "EventAwareLumiBased", "type" : str, "optional" : True, "validate" : lambda x: x in ["EventBased", "LumiBased", "EventAwareLumiBased", "FileBased"], "attr" : "procJobSplitAlgo", "null" : False}, "EventsPerJob" : {"default" : None, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "eventsPerJob", "null" : True}, "LumisPerJob" : {"default" : 8, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "lumisPerJob", "null" : False}, "FilesPerJob" : {"default" : 1, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "filesPerJob", "null" : False} } baseArgs.update(specArgs) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr() baseArgs.update(reqMgrArgs) specArgs = {"RequestType" : {"default" : "StoreResults", "optional" : True, "attr" : "requestType"}, "InputDataset" : {"default" : None, "type" : str, "optional" : False, "validate" : dataset, "attr" : "inputDataset", "null" : False}, "GlobalTag" : {"default" : "GT_SR_V1:All", "type" : str, "optional" : False, "validate" : None, "attr" : "globalTag", "null" : False}, "CmsPath" : {"default" : "/tmp", "type" : str, "optional" : False, "validate" : None, "attr" : "cmsPath", "null" : False}, "DataTier" : {"default" : "USER", "type" : str, "optional" : True, "validate" : None, "attr" : "dataTier", "null" : False}, "UnmergedLFNBase" : {"default" : "/store/unmerged", "type" : str, "optional" : True, "validate" : None, "attr" : "unmergedLFNBase", "null" : False}, "MergedLFNBase" : {"default" : "/store/results", "type" : str, "optional" : True, "validate" : None, "attr" : "mergedLFNBase", "null" : False}, "MinMergeSize" : {"default" : 2 * 1024 * 1024 * 1024, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "minMergeSize", "null" : False}, "MaxMergeSize" : {"default" : 4 * 1024 * 1024 * 1024, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "maxMergeSize", "null" : False}, "MaxMergeEvents" : {"default" : 100000, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "maxMergeEvents", "null" : False}, "BlockBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockBlacklist", "null" : False}, "BlockWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockWhitelist", "null" : False}, "RunBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runBlacklist", "null" : False}, "RunWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runWhitelist", "null" : False}} baseArgs.update(specArgs) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr() baseArgs.update(reqMgrArgs) specArgs = { "InputDataset": { "default": "/MinimumBias/ComissioningHI-v1/RAW", "type": str, "optional": False, "validate": dataset, "attr": "inputDataset", "null": False }, "GlobalTag": { "default": "GT_DP_V1", "type": str, "optional": False, "validate": None, "attr": "globalTag", "null": False }, "OpenRunningTimeout": { "default": 0, "type": int, "optional": True, "validate": lambda x: x >= 0, "attr": "openRunningTimeout", "null": False }, "BlockBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockBlacklist", "null": False }, "BlockWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockWhitelist", "null": False }, "RunBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runBlacklist", "null": False }, "RunWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runWhitelist", "null": False }, "SplittingAlgo": { "default": "EventAwareLumiBased", "type": str, "optional": True, "validate": lambda x: x in [ "EventBased", "LumiBased", "EventAwareLumiBased", "FileBased" ], "attr": "procJobSplitAlgo", "null": False }, "EventsPerJob": { "default": None, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "eventsPerJob", "null": True }, "LumisPerJob": { "default": 8, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "lumisPerJob", "null": False }, "FilesPerJob": { "default": 1, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "filesPerJob", "null": False } } baseArgs.update(specArgs) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() specArgs = { "RequestType": { "default": "PromptReco", "optional": True, "attr": "requestType" }, "Scenario": { "default": None, "type": str, "optional": False, "validate": None, "attr": "procScenario", "null": False }, "GlobalTag": { "default": None, "type": str, "optional": False, "validate": None, "attr": "globalTag", "null": False }, "ProcessingString": { "default": "", "validate": procstringT0 }, "WriteTiers": { "default": ["RECO", "AOD", "DQM", "ALCARECO"], "type": makeList, "optional": False, "validate": None, "attr": "writeTiers", "null": False }, "AlcaSkims": { "default": ["TkAlCosmics0T", "MuAlGlobalCosmics", "HcalCalHOCosmics"], "type": makeList, "optional": False, "validate": None, "attr": "alcaSkims", "null": False }, "InputDataset": { "default": "/Cosmics/Run2012A-v1/RAW", "type": str, "optional": False, "validate": dataset, "attr": "inputDataset", "null": False }, "PhysicsSkims": { "default": [], "type": makeList, "optional": True, "validate": None, "attr": "physicsSkims", "null": False }, "InitCommand": { "default": None, "type": str, "optional": True, "validate": None, "attr": "initCommand", "null": True }, "EnvPath": { "default": None, "type": str, "optional": True, "validate": None, "attr": "envPath", "null": True }, "BinPath": { "default": None, "type": str, "optional": True, "validate": None, "attr": "binPath", "null": True }, "DoLogCollect": { "default": True, "type": strToBool, "optional": True, "validate": None, "attr": "doLogCollect", "null": False }, "BlockBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockBlacklist", "null": False }, "BlockWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockWhitelist", "null": False }, "RunBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runBlacklist", "null": False }, "RunWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runWhitelist", "null": False }, "SplittingAlgo": { "default": "EventBased", "type": str, "optional": True, "validate": lambda x: x in [ "EventBased", "LumiBased", "EventAwareLumiBased", "FileBased" ], "attr": "procJobSplitAlgo", "null": False }, "EventsPerJob": { "default": 500, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "eventsPerJob", "null": False }, "LumisPerJob": { "default": 8, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "lumisPerJob", "null": False }, "FilesPerJob": { "default": 1, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "filesPerJob", "null": False }, "SkimSplittingAlgo": { "default": "FileBased", "type": str, "optional": True, "validate": lambda x: x in [ "EventBased", "LumiBased", "EventAwareLumiBased", "FileBased" ], "attr": "skimJobSplitAlgo", "null": False }, "SkimEventsPerJob": { "default": 500, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "skimEventsPerJob", "null": False }, "SkimLumisPerJob": { "default": 8, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "skimLumisPerJob", "null": False }, "SkimFilesPerJob": { "default": 1, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "skimFilesPerJob", "null": False }, "BlockCloseDelay": { "default": 86400, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "blockCloseDelay", "null": False } } baseArgs.update(specArgs) # add more optional arguments in case it is created using ReqMgr (not T0 case but should support both) reqMgrArguments = { "CouchURL": { "default": "https://cmsweb.cern.ch/couchdb", "validate": couchurl }, "CouchDBName": { "default": "reqmgr_config_cache", "type": str, "validate": identifier }, "ConfigCacheUrl": { "default": "https://cmsweb.cern.ch/couchdb", "validate": couchurl }, "ConfigCacheID": { "optional": True, "null": True }, "CouchWorkloadDBName": { "default": "reqmgr_workload_cache", "validate": identifier }, } baseArgs.update(reqMgrArguments) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getTaskArguments(firstTask=False, generator=False): """ _getTaskArguments_ Each task dictionary specifies its own set of arguments that need to be validated as well, most of them are already defined in StdBase.getWorkloadArguments and those do not appear here since they are all optional. Here only new arguments are listed. """ specArgs = {"TaskName": {"default": None, "type": str, "optional": False, "validate": None, "null": False}, "ConfigCacheUrl": {"default": "https://cmsweb.cern.ch/couchdb", "type": str, "optional": False, "validate": None, "attr": "configCacheUrl", "null": False}, "ConfigCacheID": {"default": None, "type": str, "optional": False, "validate": None, "null": False}, "KeepOutput": {"default": True, "type": strToBool, "optional": True, "validate": None, "null": False}, "TransientOutputModules": {"default": [], "type": makeList, "optional": True, "validate": None, "null": False}, "PrimaryDataset": {"default": None, "type": str, "optional": not generator, "validate": primdataset, "null": False}, "Seeding": {"default": "AutomaticSeeding", "type": str, "optional": True, "validate": lambda x: x in ["ReproducibleSeeding", "AutomaticSeeding"], "null": False}, "RequestNumEvents": {"default": 1000, "type": int, "optional": not generator, "validate": lambda x: x > 0, "null": False}, "MCPileup": {"default": None, "type": str, "optional": True, "validate": dataset, "null": False}, "DataPileup": {"default": None, "type": str, "optional": True, "validate": dataset, "null": False}, "DeterministicPileup": {"default": False, "type": strToBool, "optional": True, "validate": None, "attr": "deterministicPileup", "null": False}, "InputDataset": {"default": None, "type": str, "optional": generator or not firstTask, "validate": dataset, "null": False}, "InputTask": {"default": None, "type": str, "optional": firstTask, "validate": None, "null": False}, "InputFromOutputModule": {"default": None, "type": str, "optional": firstTask, "validate": None, "null": False}, "BlockBlacklist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "null": False}, "BlockWhitelist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "null": False}, "RunBlacklist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "null": False}, "RunWhitelist": {"default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "null": False}, "SplittingAlgo": {"default": "EventAwareLumiBased", "type": str, "optional": True, "validate": lambda x: x in ["EventBased", "LumiBased", "EventAwareLumiBased", "FileBased"], "null": False}, "EventsPerJob": {"default": None, "type": int, "optional": True, "validate": lambda x: x > 0, "null": False}, "LumisPerJob": {"default": 8, "type": int, "optional": True, "validate": lambda x: x > 0, "null": False}, "FilesPerJob": {"default": 1, "type": int, "optional": True, "validate": lambda x: x > 0, "null": False}, "EventsPerLumi": {"default": None, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "eventsPerLumi", "null": True}, "FilterEfficiency": {"default": 1.0, "type": float, "optional": True, "validate": lambda x: x > 0.0, "attr": "filterEfficiency", "null": False}, "LheInputFiles": {"default": False, "type": strToBool, "optional": True, "validate": None, "attr": "lheInputFiles", "null": False}, "PrepID": {"default": None, "type": str, "optional": True, "validate": None, "attr": "prepID", "null": True}, "Multicore": {"default": None, "type": int, "optional": True, "validate": lambda x: x > 0, "null": False}, } StdBase.setDefaultArgumentsProperty(specArgs) return specArgs
def getWorkloadArguments(): """ _getWorkloadArguments_ This represents the authorative list of request arguments that are interpreted by the current spec class. The list is formatted as a 2-level dictionary, the keys in the first level are the identifiers for the arguments processed by the current spec. The second level dictionary contains the information about that argument for validation: - default: Gives a default value if not provided, this default value usually is good enough for a standard workflow. If the argument is not optional and a default value is provided, this is only meant for test purposes. - type: A function that verifies the type of the argument, it may also cast it into the appropiate python type. If the input is not compatible with the expected type, this method must throw an exception. - optional: This boolean value indicates if the value must be provided or not by user or inherited class can overwrite with default value. - assign_optional: This boolean value indicates if the value must be provided when workflow is assinged if False. - validate: A function which validates the input after type casting, it returns True if the input is valid, it can throw exceptions on invalid input. - attr: This represents the name of the attribute corresponding to the argument in the WMSpec object. - null: This indicates if the argument can have None as its value. If above is not specifyed, automatically set by following default value - default: None - type: str - optional: True - assign_optional: True - validate: None - attr: change first letter to lower case - null: False Example: { RequestPriority : {'default' : 0, 'type' : int, 'optional' : False, 'validate' : lambda x : x > 0, 'attr' : 'priority', 'null' : False} } This replaces the old syntax in the __call__ of: self.priority = arguments.get("RequestPriority", 0) """ # if key is not specified it is set by default value arguments = { "RequestType": {"optional": False}, # this need to be overwritten by inherited class "Requestor": {"default": "unknown", "attr": "owner"}, "RequestorDN": {"default": "unknown", "attr": "owner_dn"}, "Group": {"default": "unknown"}, "RequestPriority": { "default": 8000, "type": int, "validate": lambda x: (x >= 0 and x < 1e6), "attr": "priority", }, "VoGroup": {"default": "unknown", "attr": "owner_vogroup"}, "VoRole": {"default": "unknown", "attr": "owner_vorole"}, "Campaign": {"default": ""}, "AcquisitionEra": {"default": "FAKE", "validate": acqname, "assign_optional": False}, "CMSSWVersion": {"validate": cmsswversion, "optional": False, "attr": "frameworkVersion"}, "ScramArch": {"default": "slc5_amd64_gcc462", "optional": False}, "GlobalTag": {"null": True}, "GlobalTagConnect": {"null": True}, "ProcessingVersion": {"default": 1, "type": int, "assign_optional": False}, "ProcessingString": {"default": "", "validate": procstring, "assign_optional": False}, "LumiList": {"default": {}, "type": makeLumiList}, "SiteBlacklist": {"default": [], "type": makeList, "validate": lambda x: all([cmsname(y) for y in x])}, "SiteWhitelist": {"default": [], "type": makeList, "validate": lambda x: all([cmsname(y) for y in x])}, "BlockBlacklist": {"default": [], "type": makeList, "validate": lambda x: all([block(y) for y in x])}, "BlockWhitelist": {"default": [], "type": makeList, "validate": lambda x: all([block(y) for y in x])}, "UnmergedLFNBase": {"default": "/store/unmerged"}, "MergedLFNBase": {"default": "/store/data"}, "MinMergeSize": {"default": 2 * 1024 * 1024 * 1024, "type": int, "validate": lambda x: x > 0}, "MaxMergeSize": {"default": 4 * 1024 * 1024 * 1024, "type": int, "validate": lambda x: x > 0}, "MaxWaitTime": {"default": 24 * 3600, "type": int, "validate": lambda x: x > 0}, "MaxMergeEvents": {"default": 100000, "type": int, "validate": lambda x: x > 0}, "ValidStatus": {"default": "PRODUCTION"}, "DbsUrl": { "default": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", "null": True, "validate": checkDBSURL, }, "DashboardHost": {"default": "cms-jobmon.cern.ch"}, "DashboardPort": {"default": 8884, "type": int, "validate": lambda x: x > 0}, "OverrideCatalog": {"null": True}, "RunNumber": {"default": 0, "type": int}, "TimePerEvent": {"default": 12.0, "type": float, "validate": lambda x: x > 0}, "Memory": {"default": 2300.0, "type": float, "validate": lambda x: x > 0}, "SizePerEvent": {"default": 512.0, "type": float, "validate": lambda x: x > 0}, "PeriodicHarvestInterval": {"default": 0, "type": int, "validate": lambda x: x >= 0}, "DQMHarvestUnit": {"default": "byRun", "type": str, "attr": "dqmHarvestUnit"}, "DQMUploadProxy": {"null": True, "attr": "dqmUploadProxy"}, "DQMUploadUrl": {"default": "https://cmsweb.cern.ch/dqm/dev", "attr": "dqmUploadUrl"}, "DQMSequences": {"default": [], "type": makeList, "attr": "dqmSequences"}, "DQMConfigCacheID": {"null": True, "attr": "dqmConfigCacheID"}, "EnableHarvesting": {"default": False, "type": strToBool}, "EnableNewStageout": {"default": False, "type": strToBool}, "IncludeParents": {"default": False, "type": strToBool}, "Multicore": {"default": 1, "null": True, "validate": lambda x: x == "auto" or (int(x) > 0)}, # data location management "TrustSitelists": {"default": False, "type": bool}, "TrustPUSitelists": {"default": False, "type": bool}, "AllowOpportunistic": {"default": False, "type": bool}, # from assignment: performance monitoring data "MaxRSS": {"default": 2411724, "type": int, "validate": lambda x: x > 0}, "MaxVSize": {"default": 20411724, "type": int, "validate": lambda x: x > 0}, "SoftTimeout": {"default": 129600, "type": int, "validate": lambda x: x > 0}, "GracePeriod": {"default": 300, "type": int, "validate": lambda x: x > 0}, # Set phedex subscription information "CustodialSites": { "default": [], "type": makeList, "assign_optional": True, "validate": lambda x: all([cmsname(y) for y in x]), }, "NonCustodialSites": { "default": [], "type": makeList, "assign_optional": True, "validate": lambda x: all([cmsname(y) for y in x]), }, "AutoApproveSubscriptionSites": { "default": [], "type": makeList, "assign_optional": True, "validate": lambda x: all([cmsname(y) for y in x]), }, # should be Low, Normal or High "SubscriptionPriority": { "default": "Low", "assign_optional": True, "validate": lambda x: x in ["Low", "Normal", "High"], }, # should be Move Replica "CustodialSubType": { "default": "Replica", "type": str, "assign_optional": True, "validate": lambda x: x in ["Move", "Replica"], }, "NonCustodialSubType": { "default": "Replica", "type": str, "assign_optional": True, "validate": lambda x: x in ["Move", "Replica"], }, # should be a valid PhEDEx group "CustodialGroup": {"default": "DataOps", "type": str, "assign_optional": True}, "NonCustodialGroup": {"default": "DataOps", "type": str, "assign_optional": True}, # should be True or False "DeleteFromSource": {"default": False, "type": bool}, # Block closing information "BlockCloseMaxWaitTime": {"default": 66400, "type": int, "validate": lambda x: x > 0}, "BlockCloseMaxFiles": {"default": 500, "type": int, "validate": lambda x: x > 0}, "BlockCloseMaxEvents": {"default": 25000000, "type": int, "validate": lambda x: x > 0}, "BlockCloseMaxSize": {"default": 5000000000000, "type": int, "validate": lambda x: x > 0}, # dashboard activity "Dashboard": {"default": "", "type": str}, # team name "Team": {"default": "", "type": str}, "PrepID": {"default": None, "null": True}, "RobustMerge": {"default": True, "type": bool}, } # Set defaults for the argument specification StdBase.setDefaultArgumentsProperty(arguments) return arguments
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() specArgs = {"InputDataset" : {"default" : "/MinimumBias/Run2012D-v1/RAW", "type" : str, "optional" : False, "validate" : dataset, "attr" : "inputDataset", "null" : False}, "GlobalTag" : {"default" : "GT_DP_V1", "type" : str, "optional" : False, "validate" : None, "attr" : "globalTag", "null" : False}, "CouchURL" : {"default" : "http://localhost:5984", "type" : str, "optional" : False, "validate" : couchurl, "attr" : "couchURL", "null" : False}, "CouchDBName" : {"default" : "dp_configcache", "type" : str, "optional" : False, "validate" : identifier, "attr" : "couchDBName", "null" : False}, "ConfigCacheUrl" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "configCacheUrl", "null" : True}, "OpenRunningTimeout" : {"default" : 0, "type" : int, "optional" : True, "validate" : lambda x : x >= 0, "attr" : "openRunningTimeout", "null" : False}, "BlockBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockBlacklist", "null" : False}, "BlockWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockWhitelist", "null" : False}, "RunBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runBlacklist", "null" : False}, "RunWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runWhitelist", "null" : False}, "SplittingAlgo" : {"default" : "EventAwareLumiBased", "type" : str, "optional" : True, "validate" : lambda x: x in ["EventBased", "LumiBased", "EventAwareLumiBased", "FileBased"], "attr" : "procJobSplitAlgo", "null" : False}, "EventsPerJob" : {"default" : None, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "eventsPerJob", "null" : False}, "LumisPerJob" : {"default" : 8, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "lumisPerJob", "null" : False}, "FilesPerJob" : {"default" : 1, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "filesPerJob", "null" : False} } baseArgs.update(specArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() specArgs = {"Scenario" : {"default" : "pp", "type" : str, "optional" : False, "validate" : None, "attr" : "procScenario", "null" : False}, "GlobalTag" : {"default" : "GR_P_V29::All", "type" : str, "optional" : False, "validate" : None, "attr" : "globalTag", "null" : False}, "WriteTiers" : {"default" : ["RECO", "AOD", "DQM", "ALCARECO"], "type" : makeList, "optional" : False, "validate" : None, "attr" : "writeTiers", "null" : False}, "AlcaSkims" : {"default" : ["TkAlCosmics0T","MuAlGlobalCosmics","HcalCalHOCosmics"], "type" : makeList, "optional" : False, "validate" : None, "attr" : "alcaSkims", "null" : False}, "InputDataset" : {"default" : "/Cosmics/Run2012A-v1/RAW", "type" : str, "optional" : False, "validate" : dataset, "attr" : "inputDataset", "null" : False}, "PromptSkims" : {"default" : [], "type" : makeList, "optional" : True, "validate" : None, "attr" : "promptSkims", "null" : False}, "CouchURL" : {"default" : None, "type" : str, "optional" : False, "validate" : couchurl, "attr" : "couchURL", "null" : False}, "CouchDBName" : {"default" : "promptreco_t", "type" : str, "optional" : False, "validate" : identifier, "attr" : "couchDBName", "null" : False}, "ConfigCacheUrl" : {"default" : None, "type" : str, "optional" : True, "validate" : couchurl, "attr" : "configCacheUrl", "null" : False}, "InitCommand" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "initCommand", "null" : False}, "EnvPath" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "envPath", "null" : True}, "BinPath" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "binPath", "null" : True}, "DoLogCollect" : {"default" : True, "type" : strToBool, "optional" : True, "validate" : None, "attr" : "doLogCollect", "null" : False}, "BlockBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockBlacklist", "null" : False}, "BlockWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockWhitelist", "null" : False}, "RunBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runBlacklist", "null" : False}, "RunWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runWhitelist", "null" : False}, "SplittingAlgo" : {"default" : "EventBased", "type" : str, "optional" : True, "validate" : lambda x: x in ["EventBased", "LumiBased", "EventAwareLumiBased", "FileBased"], "attr" : "procJobSplitAlgo", "null" : False}, "EventsPerJob" : {"default" : 500, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "eventsPerJob", "null" : False}, "LumisPerJob" : {"default" : 8, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "lumisPerJob", "null" : False}, "FilesPerJob" : {"default" : 1, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "filesPerJob", "null" : False}, "SkimSplittingAlgo" : {"default" : "FileBased", "type" : str, "optional" : True, "validate" : lambda x: x in ["EventBased", "LumiBased", "EventAwareLumiBased", "FileBased"], "attr" : "skimJobSplitAlgo", "null" : False}, "SkimEventsPerJob" : {"default" : 500, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "skimEventsPerJob", "null" : False}, "SkimLumisPerJob" : {"default" : 8, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "skimLumisPerJob", "null" : False}, "SkimFilesPerJob" : {"default" : 1, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "skimFilesPerJob", "null" : False}, "BlockCloseDelay" : {"default" : 86400, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "blockCloseDelay", "null" : False} } baseArgs.update(specArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() del baseArgs["AcquisitionEra"] reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr() baseArgs.update(reqMgrArgs) specArgs = { "RequestType": { "default": "Analysis", "optional": True, "attr": "requestType" }, "userSandbox": { "default": "http://home.fnal.gov/~ewv/agent.tgz", "type": str, "optional": True, "validate": None, "attr": "userSandbox", "null": True }, "userFiles": { "default": [], "type": makeList, "optional": True, "validate": None, "attr": "userFiles", "null": False }, "OutputFiles": { "default": [], "type": makeList, "optional": True, "validate": None, "attr": "outputFiles", "null": False }, "Username": { "default": "mmascher", "type": str, "optional": True, "validate": None, "attr": "userName", "null": False }, "SaveLogs": { "default": True, "type": strToBool, "optional": True, "validate": None, "attr": "saveLogs", "null": False }, "CouchURL": { "default": "http://localhost:5984", "type": str, "optional": False, "validate": couchurl, "attr": "couchURL", "null": False }, "CouchDBName": { "default": "analysis_reqmgr_config_cache", "type": str, "optional": False, "validate": identifier, "attr": "couchDBName", "null": False }, "AnalysisConfigCacheDoc": { "default": None, "type": str, "optional": False, "validate": None, "attr": "configCacheID", "null": False }, "ConfigCacheUrl": { "default": None, "type": str, "optional": True, "validate": None, "attr": "configCacheUrl", "null": False }, "PublishDataName": { "default": None, "type": str, "optional": True, "validate": None, "attr": "acquisitionEra", "null": False }, "GlobalTag": { "default": "GT_AN_V1:All", "type": str, "optional": False, "validate": None, "attr": "globalTag", "null": False }, "InputDataset": { "default": "/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO", "type": str, "optional": False, "validate": None, "attr": "inputDataset", "null": False }, "OriginalRequestName": { "default": None, "type": str, "optional": True, "validate": None, "attr": "origRequest", "null": False }, "BlockBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockBlacklist", "null": False }, "BlockWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockWhitelist", "null": False }, "RunBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runBlacklist", "null": False }, "RunWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runWhitelist", "null": False }, "asyncDest": { "default": "T1_US_FNAL_Buffer", "type": str, "optional": True, "validate": None, "attr": "asyncDest", "null": False }, "ACDCUrl": { "default": "http://cmsweb-testbed.cern.ch/couchdb", "type": str, "optional": True, "validate": couchurl, "attr": "ACDCURL", "null": False }, "ACDCDBName": { "default": "acdcserver", "type": str, "optional": True, "validate": identifier, "attr": "ACDCDBName", "null": False }, "Runs": { "default": None, "type": makeList, "optional": True, "validate": None, "attr": "Runs", "null": False }, "Lumis": { "default": None, "type": makeList, "optional": True, "validate": None, "attr": "Lumis", "null": False }, "Submission": { "default": 1, "type": int, "optional": True, "validate": None, "attr": "Submission", "null": False }, "JobSplitAlgo": { "default": "EventBased", "type": str, "optional": True, "validate": None, "attr": "analysisJobSplitAlgo", "null": False }, "EventsPerJob": { "default": 1000, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "eventsPerJob", "null": False }, "LumisPerJob": { "default": 8, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "lumisPerJob", "null": False }, } baseArgs.update(specArgs) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadCreateArgs(): baseArgs = StdBase.getWorkloadCreateArgs() specArgs = { "RequestType": { "default": "StoreResults", "optional": False }, "InputDataset": { "optional": False, "validate": dataset, "null": False }, "CmsPath": { "default": "/tmp", "type": str, "optional": False, "validate": None, "attr": "cmsPath", "null": False }, "DataTier": { "default": "USER", "type": str, "optional": True, "validate": None, "attr": "dataTier", "null": False }, "MergedLFNBase": { "default": "/store/results", "type": str, "optional": True, "validate": None, "attr": "mergedLFNBase", "null": False }, "BlockBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockBlacklist", "null": False }, "BlockWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockWhitelist", "null": False }, "RunBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runBlacklist", "null": False }, "RunWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runWhitelist", "null": False } } baseArgs.update(specArgs) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() specArgs = {"RequestType" : {"default" : "PromptReco", "optional" : True, "attr" : "requestType"}, "Scenario" : {"default" : None, "type" : str, "optional" : False, "validate" : None, "attr" : "procScenario", "null" : False}, "GlobalTag" : {"default" : None, "type" : str, "optional" : False, "validate" : None, "attr" : "globalTag", "null" : False}, "ProcessingString": {"default": "", "validate": procstringT0}, "WriteTiers" : {"default" : ["RECO", "AOD", "DQM", "ALCARECO"], "type" : makeList, "optional" : False, "validate" : None, "attr" : "writeTiers", "null" : False}, "AlcaSkims" : {"default" : ["TkAlCosmics0T","MuAlGlobalCosmics","HcalCalHOCosmics"], "type" : makeList, "optional" : False, "validate" : None, "attr" : "alcaSkims", "null" : False}, "InputDataset" : {"default" : "/Cosmics/Run2012A-v1/RAW", "type" : str, "optional" : False, "validate" : dataset, "attr" : "inputDataset", "null" : False}, "PhysicsSkims" : {"default" : [], "type" : makeList, "optional" : True, "validate" : None, "attr" : "physicsSkims", "null" : False}, "InitCommand" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "initCommand", "null" : True}, "EnvPath" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "envPath", "null" : True}, "BinPath" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "binPath", "null" : True}, "DoLogCollect" : {"default" : True, "type" : strToBool, "optional" : True, "validate" : None, "attr" : "doLogCollect", "null" : False}, "BlockBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockBlacklist", "null" : False}, "BlockWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockWhitelist", "null" : False}, "RunBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runBlacklist", "null" : False}, "RunWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runWhitelist", "null" : False}, "SplittingAlgo" : {"default" : "EventBased", "type" : str, "optional" : True, "validate" : lambda x: x in ["EventBased", "LumiBased", "EventAwareLumiBased", "FileBased"], "attr" : "procJobSplitAlgo", "null" : False}, "EventsPerJob" : {"default" : 500, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "eventsPerJob", "null" : False}, "LumisPerJob" : {"default" : 8, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "lumisPerJob", "null" : False}, "FilesPerJob" : {"default" : 1, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "filesPerJob", "null" : False}, "SkimSplittingAlgo" : {"default" : "FileBased", "type" : str, "optional" : True, "validate" : lambda x: x in ["EventBased", "LumiBased", "EventAwareLumiBased", "FileBased"], "attr" : "skimJobSplitAlgo", "null" : False}, "SkimEventsPerJob" : {"default" : 500, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "skimEventsPerJob", "null" : False}, "SkimLumisPerJob" : {"default" : 8, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "skimLumisPerJob", "null" : False}, "SkimFilesPerJob" : {"default" : 1, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "skimFilesPerJob", "null" : False}, "BlockCloseDelay" : {"default" : 86400, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "blockCloseDelay", "null" : False} } baseArgs.update(specArgs) # add more optional arguments in case it is created using ReqMgr (not T0 case but should support both) reqMgrArguments = {"CouchURL" : {"default" : "https://cmsweb.cern.ch/couchdb", "validate" : couchurl}, "CouchDBName" : {"default" : "reqmgr_config_cache", "type" : str, "validate" : identifier}, "ConfigCacheUrl" : {"default" :"https://cmsweb.cern.ch/couchdb", "validate" : couchurl}, "ConfigCacheID" : {"optional": True, "null": True}, "CouchWorkloadDBName" : {"default" : "reqmgr_workload_cache", "validate" : identifier}, } baseArgs.update(reqMgrArguments) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() del baseArgs["AcquisitionEra"] reqMgrArgs = StdBase.getWorkloadArgumentsWithReqMgr() baseArgs.update(reqMgrArgs) specArgs = {"RequestType" : {"default" : "Analysis", "optional" : True, "attr" : "requestType"}, "userSandbox" : {"default" : "http://home.fnal.gov/~ewv/agent.tgz", "type" : str, "optional" : True, "validate" : None, "attr" : "userSandbox", "null" : True}, "userFiles" : {"default" : [], "type" : makeList, "optional" : True, "validate" : None, "attr" : "userFiles", "null" : False}, "OutputFiles" : {"default" : [], "type" : makeList, "optional" : True, "validate" : None, "attr" : "outputFiles", "null" : False}, "Username" : {"default" : "mmascher", "type" : str, "optional" : True, "validate" : None, "attr" : "userName", "null" : False}, "SaveLogs" : {"default" : True, "type" : strToBool, "optional" : True, "validate" : None, "attr" : "saveLogs", "null" : False}, "CouchURL" : {"default" : "http://localhost:5984", "type" : str, "optional" : False, "validate" : couchurl, "attr" : "couchURL", "null" : False}, "CouchDBName" : {"default" : "analysis_reqmgr_config_cache", "type" : str, "optional" : False, "validate" : identifier, "attr" : "couchDBName", "null" : False}, "AnalysisConfigCacheDoc" : {"default" : None, "type" : str, "optional" : False, "validate" : None, "attr" : "configCacheID", "null" : False}, "ConfigCacheUrl" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "configCacheUrl", "null" : False}, "PublishDataName" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "acquisitionEra", "null" : False}, "GlobalTag" : {"default" : "GT_AN_V1:All", "type" : str, "optional" : False, "validate" : None, "attr" : "globalTag", "null" : False}, "InputDataset" : {"default" : "/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO", "type" : str, "optional" : False, "validate" : None, "attr" : "inputDataset", "null" : False}, "OriginalRequestName" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "origRequest", "null" : False}, "BlockBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockBlacklist", "null" : False}, "BlockWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockWhitelist", "null" : False}, "RunBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runBlacklist", "null" : False}, "RunWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runWhitelist", "null" : False}, "asyncDest" : {"default" : "T1_US_FNAL_Buffer", "type" : str, "optional" : True, "validate" : None, "attr" : "asyncDest", "null" : False}, "ACDCUrl" : {"default" : "http://cmsweb-testbed.cern.ch/couchdb", "type" : str, "optional" : True, "validate" : couchurl, "attr" : "ACDCURL", "null" : False}, "ACDCDBName" : {"default" : "acdcserver", "type" : str, "optional" : True, "validate" : identifier, "attr" : "ACDCDBName", "null" : False}, "Runs" : {"default" : None, "type" : makeList, "optional" : True, "validate" : None, "attr" : "Runs", "null" : False}, "Lumis" : {"default" : None, "type" : makeList, "optional" : True, "validate" : None, "attr" : "Lumis", "null" : False}, "Submission" : {"default" : 1, "type" : int, "optional" : True, "validate" : None, "attr" : "Submission", "null" : False}, "JobSplitAlgo" : {"default" : "EventBased", "type" : str, "optional" : True, "validate" : None, "attr" : "analysisJobSplitAlgo", "null" : False}, "EventsPerJob" : {"default" : 1000, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "eventsPerJob", "null" : False}, "LumisPerJob" : {"default" : 8, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "lumisPerJob", "null" : False}, } baseArgs.update(specArgs) StdBase.setDefaultArgumentsProperty(baseArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() specArgs = {"Scenario" : {"default" : None, "type" : str, "optional" : False, "validate" : None, "attr" : "procScenario", "null" : False}, "GlobalTag" : {"default" : None, "type" : str, "optional" : False, "validate" : None, "attr" : "globalTag", "null" : False}, "WriteTiers" : {"default" : ["RECO", "AOD", "DQM", "ALCARECO"], "type" : makeList, "optional" : False, "validate" : None, "attr" : "writeTiers", "null" : False}, "AlcaSkims" : {"default" : ["TkAlCosmics0T","MuAlGlobalCosmics","HcalCalHOCosmics"], "type" : makeList, "optional" : False, "validate" : None, "attr" : "alcaSkims", "null" : False}, "InputDataset" : {"default" : "/Cosmics/Run2012A-v1/RAW", "type" : str, "optional" : False, "validate" : dataset, "attr" : "inputDataset", "null" : False}, "PromptSkims" : {"default" : [], "type" : makeList, "optional" : True, "validate" : None, "attr" : "promptSkims", "null" : False}, "CouchURL" : {"default" : None, "type" : str, "optional" : False, "validate" : couchurl, "attr" : "couchURL", "null" : False}, "CouchDBName" : {"default" : "promptreco_t", "type" : str, "optional" : True, "validate" : identifier, "attr" : "couchDBName", "null" : False}, "ConfigCacheUrl" : {"default" : None, "type" : str, "optional" : True, "validate" : couchurl, "attr" : "configCacheUrl", "null" : False}, "InitCommand" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "initCommand", "null" : False}, "EnvPath" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "envPath", "null" : True}, "BinPath" : {"default" : None, "type" : str, "optional" : True, "validate" : None, "attr" : "binPath", "null" : True}, "DoLogCollect" : {"default" : True, "type" : strToBool, "optional" : True, "validate" : None, "attr" : "doLogCollect", "null" : False}, "BlockBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockBlacklist", "null" : False}, "BlockWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockWhitelist", "null" : False}, "RunBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runBlacklist", "null" : False}, "RunWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runWhitelist", "null" : False}, "SplittingAlgo" : {"default" : "EventBased", "type" : str, "optional" : True, "validate" : lambda x: x in ["EventBased", "LumiBased", "EventAwareLumiBased", "FileBased"], "attr" : "procJobSplitAlgo", "null" : False}, "EventsPerJob" : {"default" : 500, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "eventsPerJob", "null" : False}, "LumisPerJob" : {"default" : 8, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "lumisPerJob", "null" : False}, "FilesPerJob" : {"default" : 1, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "filesPerJob", "null" : False}, "SkimSplittingAlgo" : {"default" : "FileBased", "type" : str, "optional" : True, "validate" : lambda x: x in ["EventBased", "LumiBased", "EventAwareLumiBased", "FileBased"], "attr" : "skimJobSplitAlgo", "null" : False}, "SkimEventsPerJob" : {"default" : 500, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "skimEventsPerJob", "null" : False}, "SkimLumisPerJob" : {"default" : 8, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "skimLumisPerJob", "null" : False}, "SkimFilesPerJob" : {"default" : 1, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "skimFilesPerJob", "null" : False}, "BlockCloseDelay" : {"default" : 86400, "type" : int, "optional" : True, "validate" : lambda x : x > 0, "attr" : "blockCloseDelay", "null" : False} } baseArgs.update(specArgs) return baseArgs
def getTaskArguments(firstTask=False, generator=False): """ _getTaskArguments_ Each task dictionary specifies its own set of arguments that need to be validated as well, most of them are already defined in StdBase.getWorkloadArguments and those do not appear here since they are all optional. Here only new arguments are listed. """ specArgs = { "TaskName": { "default": None, "type": str, "optional": False, "validate": None, "null": False }, "ConfigCacheUrl": { "default": "https://cmsweb.cern.ch/couchdb", "type": str, "optional": False, "validate": None, "attr": "configCacheUrl", "null": False }, "ConfigCacheID": { "default": None, "type": str, "optional": False, "validate": None, "null": False }, "KeepOutput": { "default": True, "type": strToBool, "optional": True, "validate": None, "null": False }, "TransientOutputModules": { "default": [], "type": makeList, "optional": True, "validate": None, "null": False }, "PrimaryDataset": { "default": None, "type": str, "optional": not generator, "validate": primdataset, "null": False }, "Seeding": { "default": "AutomaticSeeding", "type": str, "optional": True, "validate": lambda x: x in ["ReproducibleSeeding", "AutomaticSeeding"], "null": False }, "RequestNumEvents": { "default": 1000, "type": int, "optional": not generator, "validate": lambda x: x > 0, "null": False }, "MCPileup": { "default": None, "type": str, "optional": True, "validate": dataset, "null": False }, "DataPileup": { "default": None, "type": str, "optional": True, "validate": dataset, "null": False }, "DeterministicPileup": { "default": False, "type": strToBool, "optional": True, "validate": None, "attr": "deterministicPileup", "null": False }, "InputDataset": { "default": None, "type": str, "optional": generator or not firstTask, "validate": dataset, "null": False }, "InputTask": { "default": None, "type": str, "optional": firstTask, "validate": None, "null": False }, "InputFromOutputModule": { "default": None, "type": str, "optional": firstTask, "validate": None, "null": False }, "BlockBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "null": False }, "BlockWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "null": False }, "RunBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "null": False }, "RunWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "null": False }, "SplittingAlgo": { "default": "EventAwareLumiBased", "type": str, "optional": True, "validate": lambda x: x in [ "EventBased", "LumiBased", "EventAwareLumiBased", "FileBased" ], "null": False }, "EventsPerJob": { "default": None, "type": int, "optional": True, "validate": lambda x: x > 0, "null": False }, "LumisPerJob": { "default": 8, "type": int, "optional": True, "validate": lambda x: x > 0, "null": False }, "FilesPerJob": { "default": 1, "type": int, "optional": True, "validate": lambda x: x > 0, "null": False }, "EventsPerLumi": { "default": None, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "eventsPerLumi", "null": True }, "FilterEfficiency": { "default": 1.0, "type": float, "optional": True, "validate": lambda x: x > 0.0, "attr": "filterEfficiency", "null": False }, "LheInputFiles": { "default": False, "type": strToBool, "optional": True, "validate": None, "attr": "lheInputFiles", "null": False }, "PrepID": { "default": None, "type": str, "optional": True, "validate": None, "attr": "prepID", "null": True }, "Multicore": { "default": None, "type": int, "optional": True, "validate": lambda x: x > 0, "null": False }, } StdBase.setDefaultArgumentsProperty(specArgs) return specArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() specArgs = { "InputDataset": { "default": "/MinimumBias/Run2010A-Dec22ReReco_v1/USER", "type": str, "optional": False, "validate": dataset, "attr": "inputDataset", "null": False, }, "GlobalTag": { "default": "GT_SR_V1:All", "type": str, "optional": False, "validate": None, "attr": "globalTag", "null": False, }, "CmsPath": { "default": "/tmp", "type": str, "optional": False, "validate": None, "attr": "cmsPath", "null": False, }, "DataTier": { "default": "USER", "type": str, "optional": True, "validate": None, "attr": "dataTier", "null": False, }, "UnmergedLFNBase": { "default": "/store/unmerged", "type": str, "optional": True, "validate": None, "attr": "unmergedLFNBase", "null": False, }, "MergedLFNBase": { "default": "/store/results", "type": str, "optional": True, "validate": None, "attr": "mergedLFNBase", "null": False, }, "MinMergeSize": { "default": 2 * 1024 * 1024 * 1024, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "minMergeSize", "null": False, }, "MaxMergeSize": { "default": 4 * 1024 * 1024 * 1024, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "maxMergeSize", "null": False, }, "MaxMergeEvents": { "default": 100000, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "maxMergeEvents", "null": False, }, "BlockBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockBlacklist", "null": False, }, "BlockWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockWhitelist", "null": False, }, "RunBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runBlacklist", "null": False, }, "RunWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runWhitelist", "null": False, }, } baseArgs.update(specArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() specArgs = { "InputDataset": { "default": "/MinimumBias/ComissioningHI-v1/RAW", "type": str, "optional": False, "validate": dataset, "attr": "inputDataset", "null": False }, "GlobalTag": { "default": "GT_DP_V1", "type": str, "optional": False, "validate": None, "attr": "globalTag", "null": False }, "CouchURL": { "default": "http://localhost:5984", "type": str, "optional": False, "validate": couchurl, "attr": "couchURL", "null": False }, "CouchDBName": { "default": "dp_configcache", "type": str, "optional": False, "validate": identifier, "attr": "couchDBName", "null": False }, "ConfigCacheUrl": { "default": None, "type": str, "optional": True, "validate": None, "attr": "configCacheUrl", "null": True }, "OpenRunningTimeout": { "default": 0, "type": int, "optional": True, "validate": lambda x: x >= 0, "attr": "openRunningTimeout", "null": False }, "BlockBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockBlacklist", "null": False }, "BlockWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([block(y) for y in x]), "attr": "blockWhitelist", "null": False }, "RunBlacklist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runBlacklist", "null": False }, "RunWhitelist": { "default": [], "type": makeList, "optional": True, "validate": lambda x: all([int(y) > 0 for y in x]), "attr": "runWhitelist", "null": False }, "SplittingAlgo": { "default": "EventAwareLumiBased", "type": str, "optional": True, "validate": lambda x: x in [ "EventBased", "LumiBased", "EventAwareLumiBased", "FileBased" ], "attr": "procJobSplitAlgo", "null": False }, "EventsPerJob": { "default": None, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "eventsPerJob", "null": False }, "LumisPerJob": { "default": 8, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "lumisPerJob", "null": False }, "FilesPerJob": { "default": 1, "type": int, "optional": True, "validate": lambda x: x > 0, "attr": "filesPerJob", "null": False } } baseArgs.update(specArgs) return baseArgs
def getWorkloadArguments(): baseArgs = StdBase.getWorkloadArguments() specArgs = {"InputDataset" : {"default" : "/MinimumBias/Run2010A-Dec22ReReco_v1/USER", "type" : str, "optional" : False, "validate" : dataset, "attr" : "inputDataset", "null" : False}, "GlobalTag" : {"default" : "GT_SR_V1:All", "type" : str, "optional" : False, "validate" : None, "attr" : "globalTag", "null" : False}, "CmsPath" : {"default" : "/tmp", "type" : str, "optional" : False, "validate" : None, "attr" : "cmsPath", "null" : False}, "DataTier" : {"default" : "USER", "type" : str, "optional" : True, "validate" : None, "attr" : "dataTier", "null" : False}, "BlockBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockBlacklist", "null" : False}, "BlockWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([block(y) for y in x]), "attr" : "blockWhitelist", "null" : False}, "RunBlacklist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runBlacklist", "null" : False}, "RunWhitelist" : {"default" : [], "type" : makeList, "optional" : True, "validate" : lambda x: all([int(y) > 0 for y in x]), "attr" : "runWhitelist", "null" : False}} baseArgs.update(specArgs) return baseArgs