def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion", "Requestor", "ScramArch",
"PrimaryDataset", "GlobalTag", "RequestSizeEvents",
"GenConfigCacheID", "StepOneConfigCacheID", "StepTwoConfigCacheID",
"GenOutputModuleName", "StepOneOutputModuleName"]
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion",
"AcquisitionEra",
"ScramArch",
"GlobalTag",
"TaskChain",
]
self.optionalFields = [
"SiteWhitelist",
"SiteBlacklist",
"BlockWhitelist",
"BlockBlacklist",
"RunWhitelist",
"RunBlacklist",
"ProcessingVersion",
"CouchUrl",
"CouchDBName",
"DbsUrl",
"UnmergedLFNBase",
"MergedLFNBase",
"MinMergeSize",
"MaxMergeSize",
"MaxMergeEvents"
]
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion",
"ScramArch",
"GlobalTag",
"InputDataset",
"StepOneConfigCacheID",
]
self.optionalFields = [
"StepThreeConfigCacheID",
"StepOneOutputModuleName"
"StepTwoConfigCacheID",
"StepTwoOutputModuleName",
"SiteWhitelist",
"SiteBlacklist",
"BlockWhitelist",
"BlockBlacklist",
"RunWhitelist",
"RunBlacklist",
"CouchUrl",
"CouchDBName",
"DbsUrl",
"UnmergedLFNBase",
"MergedLFNBase",
"MinMergeSize",
"MaxMergeSize",
"MaxMergeEvents"
]
def validate(self):
"""
_validate_
Check that basic required parameters are provided and do some more
detailed checks on each of the parameters for the chain sub steps
"""
RequestSchema.validate(self)
try:
numTasks = int(self['TaskChain'])
except ValueError:
msg = "TaskChain parameter is not an Integer"
raise RuntimeError(msg)
for i in range(1, numTasks + 1):
if not self.has_key("Task%s" % i):
msg = "No Task%s entry present in request" % i
raise RuntimeError, msg
if i == 1:
if self['Task1'].has_key('InputDataset'):
validateProcFirstTask(self['Task1'])
else:
validateGenFirstTask(self['Task1'])
validateSubTask(self['Task1'], firstTask=True)
else:
validateSubTask(self['Task%s' % i])
def validate(self):
"""
_validate_
Check that basic required parameters are provided and do some more
detailed checks on each of the parameters for the chain sub steps
"""
RequestSchema.validate(self)
try:
numTasks = int(self['TaskChain'])
except ValueError:
msg = "TaskChain parameter is not an Integer"
raise RuntimeError(msg)
for i in range (1, numTasks+1):
if not self.has_key("Task%s" % i):
msg = "No Task%s entry present in request" % i
raise RuntimeError, msg
if i == 1:
if self['Task1'].has_key('InputDataset'):
validateProcFirstTask(self['Task1'])
else:
validateGenFirstTask(self['Task1'])
validateSubTask(self['Task1'], firstTask = True)
else:
validateSubTask(self['Task%s' % i])
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion",
"ScramArch",
"GlobalTag",
"InputDataset"
]
self.optionalFields = [
"SiteWhitelist",
"SiteBlacklist",
"BlockWhitelist",
"BlockBlacklist",
"RunWhitelist",
"RunBlacklist",
"Scenario",
"ProcessingConfig",
"ProcessingVersion",
"SkimConfig",
"CouchUrl",
"CouchDBName",
"DbsUrl",
"UnmergedLFNBase",
"MergedLFNBase",
"MinMergeSize",
"MaxMergeSize",
"MaxMergeEvents"
]
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion", "Requestor", "ScramArch", "PrimaryDataset",
"GlobalTag", "RequestSizeEvents", "GenConfigCacheID",
"StepOneConfigCacheID", "StepTwoConfigCacheID",
"GenOutputModuleName", "StepOneOutputModuleName"
]
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("ConfigCacheID", None)
self.setdefault("PileupDataset", None)
self.validateFields = [
"CMSSWVersion", "ConfigCacheID", "PrimaryDataset"
]
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"OriginalRequestName",
"InitialTaskPath",
"ACDCServer",
"ACDCDatabase"
]
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("ProcessingVersion", None)
self.setdefault("GlobalTag", None)
self.setdefault("InputDataset", None)
self.setdefault("DBSURL", None)
self.validateFields = [
"CMSSWVersion", "ConfigCacheID", "GlobalTag", "InputDataset"
]
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("ProdConfigCacheID", None)
self.setdefault("PileupDataset", None)
self.validateFields = [
"CMSSWVersion",
"ProdConfigCacheID",
"PrimaryDataset"
]
def __init__(self):
RequestSchema.__init__(self)
# not used yet
self.validateFields = [
'InputDatasets',
'CMSSWVersion',
'ScramArch',
'Group',
'DbsUrl'
]
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion", "ScramArch", "GlobalTag", "InputDataset"
]
self.optionalFields = [
"SiteWhitelist", "SiteBlacklist", "BlockWhitelist",
"BlockBlacklist", "RunWhitelist", "RunBlacklist", "Scenario",
"ProcessingConfig", "ProcessingVersion", "SkimConfig", "CouchUrl",
"CouchDBName", "DbsUrl", "UnmergedLFNBase", "MergedLFNBase",
"MinMergeSize", "MaxMergeSize", "MaxMergeEvents"
]
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("CMSSWVersion", None)
self.setdefault("ProcessingVersion", None)
self.setdefault("GlobalTag", None)
self.setdefault("InputDataset", None)
self.setdefault("DBSURL", None)
self.validateFields = [
"CMSSWVersion",
"ConfigCacheID",
"GlobalTag",
"InputDataset"
]
def validate(self):
def _checkSiteList(list):
if self.has_key(list) and hasattr(self,'allCMSNames'):
for site in self[list]:
if not site in self.allCMSNames: #self.allCMSNames needs to be initialized to allow sitelisk check
raise RuntimeError("The site " + site + " provided in the " + list + " param has not been found. Check https://cmsweb.cern.ch/sitedb/json/index/SEtoCMSName?name= for a list of known sites")
RequestSchema.validate(self)
_checkSiteList("SiteWhitelist")
_checkSiteList("SiteBlacklist")
if self.get("RequestName").count(' ') > 0:
raise RuntimeError("RequestName cannot contain spaces")
def getNewRequestSchema(reqInputArgs):
"""
Create a new schema
"""
reqSchema = RequestSchema()
reqSchema.update(reqInputArgs)
currentTime = time.strftime("%y%m%d_%H%M%S", time.localtime(time.time()))
secondFraction = int(10000 * (time.time() % 1.0))
requestString = reqSchema.get("RequestString", "")
if requestString != "":
reqSchema["RequestName"] = "%s_%s_%s_%s" % (reqSchema["Requestor"], requestString, currentTime, secondFraction)
else:
reqSchema["RequestName"] = "%s_%s_%s" % (reqSchema["Requestor"], currentTime, secondFraction)
return reqSchema
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion",
"AcquisitionEra",
"ScramArch",
"GlobalTag",
"TaskChain",
]
self.optionalFields = [
"SiteWhitelist", "SiteBlacklist", "BlockWhitelist",
"BlockBlacklist", "RunWhitelist", "RunBlacklist",
"ProcessingVersion", "CouchUrl", "CouchDBName", "DbsUrl",
"UnmergedLFNBase", "MergedLFNBase", "MinMergeSize", "MaxMergeSize",
"MaxMergeEvents"
]
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion",
"ScramArch",
"GlobalTag",
"InputDataset",
"StepOneConfigCacheID",
]
self.optionalFields = [
"StepThreeConfigCacheID", "StepOneOutputModuleName"
"StepTwoConfigCacheID", "StepTwoOutputModuleName", "SiteWhitelist",
"SiteBlacklist", "BlockWhitelist", "BlockBlacklist",
"RunWhitelist", "RunBlacklist", "CouchUrl", "CouchDBName",
"DbsUrl", "UnmergedLFNBase", "MergedLFNBase", "MinMergeSize",
"MaxMergeSize", "MaxMergeEvents"
]
def getNewRequestSchema(reqInputArgs):
"""
Create a new schema
"""
reqSchema = RequestSchema()
reqSchema.update(reqInputArgs)
currentTime = time.strftime('%y%m%d_%H%M%S', time.localtime(time.time()))
secondFraction = int(10000 * (time.time() % 1.0))
requestString = reqSchema.get('RequestString', "")
if requestString != "":
reqSchema['RequestName'] = "%s_%s_%s_%s" % (
reqSchema['Requestor'], requestString, currentTime, secondFraction)
else:
reqSchema['RequestName'] = "%s_%s_%s" % (reqSchema['Requestor'],
currentTime, secondFraction)
return reqSchema
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"CMSSWVersion",
"GlobalTag",
"InputDatasets",
"DbsUrl",
"LFNCategory",
"OutputTiers"
]
self.optionalFields = [
"SiteWhitelist",
"SiteBlackList",
"BlockWhitelist",
"BlockBlacklist",
"RunWhitelist",
"RunBlacklist",
"Scenario"
]
def __init__(self):
RequestSchema.__init__(self)
self.setdefault("ProductionChannel", None)
self.setdefault("Label", None)
self.setdefault("Activity", "Production")
self.setdefault("CMSSWVersion", None)
self.setdefault("Category", "MC")
self.setdefault("Configuration", None)
self.setdefault("PileupDataset", None)
self.setdefault("FinalDestination", None)
self.setdefault("PSetHash", None)
self.setdefault("CmsGenConfiguration", None)
self.setdefault("CmsGenParameters", None)
self.setdefault("CmsGenSelectionEfficiency", None)
self.validateFields = [
"ProductionChannel",
"CMSSWVersion",
"Label",
"Configuration",
"PSetHash",
"CmsGenConfiguration",
"CmsGenParameters",
]
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = []
self.optionalFields = []
self.validateFields = [
"CMSSWVersion",
"ScramArch",
"InputDataset",
"Requestor",
"RequestorDN"
]
self.optionalFields = [
"SiteWhitelist",
"SiteBlacklist",
"BlockWhitelist",
"BlockBlacklist",
"RunWhitelist",
"RunBlacklist",
"CouchUrl",
"CouchDBName",
"DbsUrl",
]
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = []
self.optionalFields = []
self.validateFields = [
"CMSSWVersion",
"ScramArch",
"InputDataset",
"Requestor",
"RequestorDN"
]
self.optionalFields = [
"SiteWhitelist",
"SiteBlacklist",
"BlockWhitelist",
"BlockBlacklist",
"RunWhitelist",
"RunBlacklist",
"CouchUrl",
"CouchDBName",
"DbsUrl",
]
def makeRequest(kwargs, couchUrl, couchDB, wmstatUrl):
logging.info(kwargs)
""" Handles the submission of requests """
# make sure no extra spaces snuck in
for k, v in kwargs.iteritems():
if isinstance(v, str):
kwargs[k] = v.strip()
# Create a new schema
schema = RequestSchema()
schema.update(kwargs)
currentTime = time.strftime('%y%m%d_%H%M%S',
time.localtime(time.time()))
secondFraction = int(10000 * (time.time()%1.0))
requestString = schema.get('RequestString', "")
if requestString != "":
schema['RequestName'] = "%s_%s_%s_%s" % (
schema['Requestor'], requestString, currentTime, secondFraction)
else:
schema['RequestName'] = "%s_%s_%s" % (schema['Requestor'], currentTime, secondFraction)
schema["Campaign"] = kwargs.get("Campaign", "")
if 'ProcScenario' in kwargs and 'ProcConfigCacheID' in kwargs:
# Use input mode to delete the unused one
inputMode = kwargs.get('inputMode', None)
if inputMode == 'scenario':
del schema['ProcConfigCacheID']
if 'EnableDQMHarvest' not in kwargs:
schema["EnableHarvesting"] = False
if kwargs.has_key("InputDataset"):
schema["InputDatasets"] = [kwargs["InputDataset"]]
if kwargs.has_key("FilterEfficiency"):
kwargs["FilterEfficiency"] = float(kwargs["FilterEfficiency"])
skimNumber = 1
# a list of dictionaries
schema["SkimConfigs"] = []
while kwargs.has_key("SkimName%s" % skimNumber):
d = {}
d["SkimName"] = kwargs["SkimName%s" % skimNumber]
d["SkimInput"] = kwargs["SkimInput%s" % skimNumber]
d["Scenario"] = kwargs["Scenario"]
if kwargs.get("Skim%sConfigCacheID" % skimNumber, None) != None:
d["ConfigCacheID"] = kwargs["Skim%sConfigCacheID" % skimNumber]
schema["SkimConfigs"].append(d)
skimNumber += 1
if kwargs.has_key("DataPileup") or kwargs.has_key("MCPileup"):
schema["PileupConfig"] = {}
if kwargs.has_key("DataPileup") and kwargs["DataPileup"] != "":
schema["PileupConfig"]["data"] = [kwargs["DataPileup"]]
if kwargs.has_key("MCPileup") and kwargs["MCPileup"] != "":
schema["PileupConfig"]["mc"] = [kwargs["MCPileup"]]
for runlist in ["RunWhitelist", "RunBlacklist"]:
if runlist in kwargs:
schema[runlist] = parseRunList(kwargs[runlist])
for blocklist in ["BlockWhitelist", "BlockBlacklist"]:
if blocklist in kwargs:
schema[blocklist] = parseBlockList(kwargs[blocklist])
if "DqmSequences" in kwargs:
schema["DqmSequences"] = parseDqmSequences(kwargs["DqmSequences"])
validate(schema)
# Get the DN
schema['RequestorDN'] = cherrypy.request.user.get('dn', 'unknown')
try:
request = buildWorkloadForRequest(typename = kwargs["RequestType"],
schema = schema)
except WMSpecFactoryException, ex:
msg = ex._message
raise HTTPError(400, "Error in Workload Validation: %s" % msg)
def __init__(self):
RequestSchema.__init__(self)
# not used yet
self.validateFields = [
'InputDatasets', 'CMSSWVersion', 'ScramArch', 'Group', 'DbsUrl'
]
def validate(self):
RequestSchema.validate(self)
def __init__(self):
RequestSchema.__init__(self)
self.validateFields = [
"OriginalRequestName", "InitialTaskPath", "ACDCServer",
"ACDCDatabase"
]
def validate(self):
RequestSchema.validate(self)
def validate(self):
RequestSchema.validate(self)
if self['InputDataset'].count('/') != 3:
raise RuntimeError, "Need three slashes in InputDataset "+self['InputDataset']
def validate(self):
RequestSchema.validate(self)
assert(isinstance(self['OutputTiers'], list))
for tier in self['OutputTiers']:
assert(tier in ['RECO', 'AOD', 'ALCA'])
def validate(self):
RequestSchema.validate(self)
if self['InputDataset'].count('/') != 3:
raise RuntimeError, "Need three slashes in InputDataset " + self[
'InputDataset']
