RequestMaker to create MC requests and workflows
"""
def __init__(self):
RequestMakerInterface.__init__(self)
def makeWorkload(self, schema):
factory = MonteCarloWorkloadFactory()
return factory(schema['RequestName'], schema).data
class MonteCarloSchema(RequestSchema):
"""
_MonteCarlo_
Data Required for a standard cmsRun MC generation request.
"""
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("ConfigCacheID", None)
self.setdefault("PileupDataset", None)
self.validateFields = [
"CMSSWVersion", "ConfigCacheID", "PrimaryDataset"
]
registerRequestType("MonteCarlo", MonteCarloRequest, MonteCarloSchema)
"""
def __init__(self):
RequestMakerInterface.__init__(self)
def makeWorkload(self, schema):
factory = PrivateMCWorkloadFactory()
return factory(schema['RequestName'], schema).data
class PrivateMCSchema(RequestSchema):
"""
_PrivateMC_
Data Required for a PrivateMC cmsRun generation request.
"""
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("AnalysisConfigCacheDoc", None)
self.setdefault("PileupDataset", None)
self.validateFields = [
"CMSSWVersion",
"AnalysisConfigCacheDoc",
"PrimaryDataset",
]
registerRequestType("PrivateMC", PrivateMCRequest, PrivateMCSchema)
"RequestorDN"
]
self.optionalFields = [
"SiteWhitelist",
"SiteBlacklist",
"BlockWhitelist",
"BlockBlacklist",
"RunWhitelist",
"RunBlacklist",
"CouchUrl",
"CouchDBName",
"DbsUrl",
]
def validate(self):
def _checkSiteList(list):
if self.has_key(list) and hasattr(self,'allCMSNames'):
for site in self[list]:
if not site in self.allCMSNames: #self.allCMSNames needs to be initialized to allow sitelisk check
raise RuntimeError("The site " + site + " provided in the " + list + " param has not been found. Check https://cmsweb.cern.ch/sitedb/json/index/SEtoCMSName?name= for a list of known sites")
RequestSchema.validate(self)
_checkSiteList("SiteWhitelist")
_checkSiteList("SiteBlacklist")
if self.get("RequestName").count(' ') > 0:
raise RuntimeError("RequestName cannot contain spaces")
registerRequestType("Analysis", AnalysisRequest, AnalysisSchema)
def makeWorkload(self, schema):
return dataProcessingWorkload(schema['RequestName'], schema).data
class DataProcessingSchema(RequestSchema):
"""
_Data_
Data Required for a standard cmsRun dataset processing request that
splits the input by file
"""
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("ProcessingVersion", None)
self.setdefault("GlobalTag", None)
self.setdefault("InputDataset", None)
self.setdefault("DBSURL", None)
self.setdefault("Multicore", None)
self.validateFields = [
"CMSSWVersion",
"ProcConfigCacheID",
"GlobalTag",
"InputDataset"
]
registerRequestType("DataProcessing", DataProcessingRequest, DataProcessingSchema)
"""
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion",
"GlobalTag",
"InputDatasets",
"DbsUrl",
"LFNCategory",
"OutputTiers"
]
self.optionalFields = [
"SiteWhitelist",
"SiteBlackList",
"BlockWhitelist",
"BlockBlacklist",
"RunWhitelist",
"RunBlacklist",
"Scenario"
]
def validate(self):
RequestSchema.validate(self)
assert(isinstance(self['OutputTiers'], list))
for tier in self['OutputTiers']:
assert(tier in ['RECO', 'AOD', 'ALCA'])
registerRequestType("Reco", RecoRequest, RecoSchema)
_validate_
Check that basic required parameters are provided and do some more
detailed checks on each of the parameters for the chain sub steps
"""
RequestSchema.validate(self)
try:
numTasks = int(self['TaskChain'])
except ValueError:
msg = "TaskChain parameter is not an Integer"
raise RuntimeError(msg)
for i in range (1, numTasks+1):
if not self.has_key("Task%s" % i):
msg = "No Task%s entry present in request" % i
raise RuntimeError, msg
if i == 1:
if self['Task1'].has_key('InputDataset'):
validateProcFirstTask(self['Task1'])
else:
validateGenFirstTask(self['Task1'])
validateSubTask(self['Task1'], firstTask = True)
else:
validateSubTask(self['Task%s' % i])
registerRequestType("TaskChain", TaskChainRequest, TaskChainSchema)
"""
_RelValMCRequest_
RequestMaker to create MC requests and workflows
"""
def __init__(self):
RequestMakerInterface.__init__(self)
def makeWorkload(self, schema):
factory = RelValMCWorkloadFactory()
return factory(schema['RequestName'], schema).data
class RelValMCSchema(RequestSchema):
"""
_RelValMC_
Data Required for a standard cmsRun MC generation request.
"""
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion", "Requestor", "ScramArch",
"PrimaryDataset", "GlobalTag", "RequestSizeEvents",
"GenConfigCacheID", "StepOneConfigCacheID", "StepTwoConfigCacheID",
"GenOutputModuleName", "StepOneOutputModuleName"]
registerRequestType("RelValMC", RelValMCRequest, RelValMCSchema)
"ScramArch",
"GlobalTag",
"InputDataset",
"StepOneConfigCacheID",
]
self.optionalFields = [
"StepThreeConfigCacheID",
"StepOneOutputModuleName"
"StepTwoConfigCacheID",
"StepTwoOutputModuleName",
"SiteWhitelist",
"SiteBlacklist",
"BlockWhitelist",
"BlockBlacklist",
"RunWhitelist",
"RunBlacklist",
"CouchUrl",
"CouchDBName",
"DbsUrl",
"UnmergedLFNBase",
"MergedLFNBase",
"MinMergeSize",
"MaxMergeSize",
"MaxMergeEvents"
]
def validate(self):
RequestSchema.validate(self)
registerRequestType("ReDigi", ReDigiRequest, ReDigiSchema)
"""
def __init__(self):
RequestMakerInterface.__init__(self)
def makeWorkload(self, schema):
factory = PrivateMCWorkloadFactory()
return factory(schema['RequestName'], schema).data
class PrivateMCSchema(RequestSchema):
"""
_PrivateMC_
Data Required for a PrivateMC cmsRun generation request.
"""
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("AnalysisConfigCacheDoc", None)
self.setdefault("PileupDataset", None)
self.validateFields = [
"CMSSWVersion",
"AnalysisConfigCacheDoc",
"PrimaryDataset",
]
registerRequestType("PrivateMC", PrivateMCRequest, PrivateMCSchema)
def __init__(self):
RequestMakerInterface.__init__(self)
def makeWorkload(self, schema):
return dataProcessingWorkload(schema['RequestName'], schema).data
class DataProcessingSchema(RequestSchema):
"""
_Data_
Data Required for a standard cmsRun dataset processing request that
splits the input by file
"""
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("ProcessingVersion", None)
self.setdefault("GlobalTag", None)
self.setdefault("InputDataset", None)
self.setdefault("DBSURL", None)
self.setdefault("Multicore", None)
self.validateFields = [
"CMSSWVersion", "ConfigCacheID", "GlobalTag", "InputDataset"
]
registerRequestType("DataProcessing", DataProcessingRequest,
DataProcessingSchema)
Data Required for a standard cmsRun MC generation request.
"""
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("ProductionChannel", None)
self.setdefault("Label", None)
self.setdefault("Activity", "Production")
self.setdefault("CMSSWVersion", None)
self.setdefault("Category", "MC")
self.setdefault("Configuration", None)
self.setdefault("PileupDataset", None)
self.setdefault("FinalDestination", None)
self.setdefault("PSetHash", None)
self.setdefault("CmsGenConfiguration", None)
self.setdefault("CmsGenParameters", None)
self.setdefault("CmsGenSelectionEfficiency", None)
self.validateFields = [
"ProductionChannel",
"CMSSWVersion",
"Label",
"Configuration",
"PSetHash",
"CmsGenConfiguration",
"CmsGenParameters",
]
registerRequestType("CmsGen", CmsGenRequest, CmsGenSchema)
from WMCore.RequestManager.RequestMaker.Registry import registerRequestType, retrieveRequestMaker
class StoreResultsRequest(RequestMakerInterface):
"""
_StoreResultsRequest_
RequestMaker to two file input data processing requests and workflows
"""
def __init__(self):
RequestMakerInterface.__init__(self)
class StoreResultsSchema(RequestSchema):
"""
_StoreResults_
Data Required for a standard cmsRun two file read processing request.
"""
def __init__(self):
RequestSchema.__init__(self)
# not used yet
self.validateFields = [
'InputDatasets', 'CMSSWVersion', 'ScramArch', 'Group', 'DbsUrl'
]
registerRequestType("StoreResults", StoreResultsRequest, StoreResultsSchema)
"""
_validate_
Check that basic required parameters are provided and do some more
detailed checks on each of the parameters for the chain sub steps
"""
RequestSchema.validate(self)
try:
numTasks = int(self['TaskChain'])
except ValueError:
msg = "TaskChain parameter is not an Integer"
raise RuntimeError(msg)
for i in range(1, numTasks + 1):
if not self.has_key("Task%s" % i):
msg = "No Task%s entry present in request" % i
raise RuntimeError, msg
if i == 1:
if self['Task1'].has_key('InputDataset'):
validateProcFirstTask(self['Task1'])
else:
validateGenFirstTask(self['Task1'])
validateSubTask(self['Task1'], firstTask=True)
else:
validateSubTask(self['Task%s' % i])
registerRequestType("TaskChain", TaskChainRequest, TaskChainSchema)
"""
def __init__(self):
RequestMakerInterface.__init__(self)
def makeWorkload(self, schema):
return monteCarloFromGENWorkload(schema['RequestName'], schema).data
class MonteCarloFromGENSchema(RequestSchema):
"""
_MonteCarloFromGENSchema_
"""
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("ProcessingVersion", None)
self.setdefault("GlobalTag", None)
self.setdefault("InputDataset", None)
self.setdefault("DBSURL", None)
self.validateFields = [
"CMSSWVersion",
"ConfigCacheID",
"GlobalTag",
"InputDataset"
]
registerRequestType("MonteCarloFromGEN", MonteCarloFromGENRequest, MonteCarloFromGENSchema)
RequestMaker to create MC requests and workflows
"""
def __init__(self):
RequestMakerInterface.__init__(self)
def makeWorkload(self, schema):
factory = RelValMCWorkloadFactory()
return factory(schema['RequestName'], schema).data
class RelValMCSchema(RequestSchema):
"""
_RelValMC_
Data Required for a standard cmsRun MC generation request.
"""
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion", "Requestor", "ScramArch", "PrimaryDataset",
"GlobalTag", "RequestSizeEvents", "GenConfigCacheID",
"StepOneConfigCacheID", "StepTwoConfigCacheID",
"GenOutputModuleName", "StepOneOutputModuleName"
]
registerRequestType("RelValMC", RelValMCRequest, RelValMCSchema)
class ReDigiSchema(RequestSchema):
"""
_ReDigi_
"""
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion",
"ScramArch",
"GlobalTag",
"InputDataset",
"StepOneConfigCacheID",
]
self.optionalFields = [
"StepThreeConfigCacheID", "StepOneOutputModuleName"
"StepTwoConfigCacheID", "StepTwoOutputModuleName", "SiteWhitelist",
"SiteBlacklist", "BlockWhitelist", "BlockBlacklist",
"RunWhitelist", "RunBlacklist", "CouchUrl", "CouchDBName",
"DbsUrl", "UnmergedLFNBase", "MergedLFNBase", "MinMergeSize",
"MaxMergeSize", "MaxMergeEvents"
]
def validate(self):
RequestSchema.validate(self)
registerRequestType("ReDigi", ReDigiRequest, ReDigiSchema)
"""
def __init__(self):
RequestMakerInterface.__init__(self)
def makeWorkload(self, schema):
factory = MonteCarloWorkloadFactory()
return factory(schema['RequestName'], schema).data
class MonteCarloSchema(RequestSchema):
"""
_MonteCarlo_
Data Required for a standard cmsRun MC generation request.
"""
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("ProdConfigCacheID", None)
self.setdefault("PileupDataset", None)
self.validateFields = [
"CMSSWVersion",
"ProdConfigCacheID",
"PrimaryDataset"
]
registerRequestType("MonteCarlo", MonteCarloRequest, MonteCarloSchema)
_ResubmissionRequest_
"""
def __init__(self):
RequestMakerInterface.__init__(self)
def makeWorkload(self, schema):
originalRequest = GetRequest.getRequestByName(
schema['OriginalRequestName'])
helper = loadWorkload(originalRequest)
helper.truncate(schema["RequestName"], schema["InitialTaskPath"],
schema["ACDCServer"], schema["ACDCDatabase"])
return helper.data
class ResubmissionSchema(RequestSchema):
"""
_ResubmissionSchema_
"""
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"OriginalRequestName", "InitialTaskPath", "ACDCServer",
"ACDCDatabase"
]
registerRequestType("Resubmission", ResubmissionRequest, ResubmissionSchema)
"""
_MonteCarloFromGENRequest_
"""
def __init__(self):
RequestMakerInterface.__init__(self)
def makeWorkload(self, schema):
return monteCarloFromGENWorkload(schema['RequestName'], schema).data
class MonteCarloFromGENSchema(RequestSchema):
"""
_MonteCarloFromGENSchema_
"""
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("ProcessingVersion", None)
self.setdefault("GlobalTag", None)
self.setdefault("InputDataset", None)
self.setdefault("DBSURL", None)
self.validateFields = [
"CMSSWVersion", "ConfigCacheID", "GlobalTag", "InputDataset"
]
registerRequestType("MonteCarloFromGEN", MonteCarloFromGENRequest,
MonteCarloFromGENSchema)
class AnalysisSchema(RequestSchema):
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = []
self.optionalFields = []
self.validateFields = [
"CMSSWVersion",
"ScramArch",
"InputDataset",
"Requestor",
"RequestorDN"
]
self.optionalFields = [
"SiteWhitelist",
"SiteBlacklist",
"BlockWhitelist",
"BlockBlacklist",
"RunWhitelist",
"RunBlacklist",
"CouchUrl",
"CouchDBName",
"DbsUrl",
]
def validate(self):
RequestSchema.validate(self)
registerRequestType("Analysis", AnalysisRequest, AnalysisSchema)
"""
_StoreResultsRequest_
RequestMaker to two file input data processing requests and workflows
"""
def __init__(self):
RequestMakerInterface.__init__(self)
class StoreResultsSchema(RequestSchema):
"""
_StoreResults_
Data Required for a standard cmsRun two file read processing request.
"""
def __init__(self):
RequestSchema.__init__(self)
# not used yet
self.validateFields = [
'InputDatasets',
'CMSSWVersion',
'ScramArch',
'Group',
'DbsUrl'
]
registerRequestType("StoreResults", StoreResultsRequest, StoreResultsSchema)
"GlobalTag",
"InputDataset"
]
self.optionalFields = [
"SiteWhitelist",
"SiteBlacklist",
"BlockWhitelist",
"BlockBlacklist",
"RunWhitelist",
"RunBlacklist",
"Scenario",
"ProcessingConfig",
"ProcessingVersion",
"SkimConfig",
"CouchUrl",
"CouchDBName",
"DbsUrl",
"UnmergedLFNBase",
"MergedLFNBase",
"MinMergeSize",
"MaxMergeSize",
"MaxMergeEvents"
]
def validate(self):
RequestSchema.validate(self)
if self['InputDataset'].count('/') != 3:
raise RuntimeError, "Need three slashes in InputDataset "+self['InputDataset']
registerRequestType("ReReco", ReRecoRequest, ReRecoSchema)
"""
_ResubmissionRequest_
"""
def __init__(self):
RequestMakerInterface.__init__(self)
def makeWorkload(self, schema):
originalRequest = GetRequest.getRequestByName(schema['OriginalRequestName'])
helper = loadWorkload(originalRequest)
helper.truncate(schema["RequestName"], schema["InitialTaskPath"],
schema["ACDCServer"], schema["ACDCDatabase"])
return helper.data
class ResubmissionSchema(RequestSchema):
"""
_ResubmissionSchema_
"""
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"OriginalRequestName",
"InitialTaskPath",
"ACDCServer",
"ACDCDatabase"
]
registerRequestType("Resubmission", ResubmissionRequest, ResubmissionSchema)
