def __call__(self, workloadName, arguments):
"""
_call_
Create a ReReco workload with the given parameters.
"""
workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments)
# The SkimConfig parameter must be a list of dictionaries where each
# dictionary will have the following keys:
# SkimName
# SkimInput
# SkimSplitAlgo - Optional at workflow creation time
# SkimSplitParams - Optional at workflow creation time
# ConfigCacheID
# Scenario
#
# The ConfigCacheID and Scenaio are mutually exclusive, only one can be
# set. The SkimSplitAlgo and SkimSplitParams don't have to be set when
# the workflow is created but must be set when the workflow is approved.
# The SkimInput is the name of the output module in the processing step
# that will be used as the input for the skim.
self.skimConfigs = arguments.get("SkimConfigs", [])
# These are mostly place holders because the job splitting algo and
# parameters will be updated after the workflow has been created.
self.skimJobSplitAlgo = arguments.get("SkimJobSplitAlgo", "FileBased")
self.skimJobSplitArgs = arguments.get("SkimJobSplitArgs", {"files_per_job": 1, "include_parents": True})
return self.addSkims(workload)
def __call__(self, workloadName, arguments):
"""
_call_
Create a ReReco workload with the given parameters.
"""
workload = DataProcessingWorkloadFactory.__call__(
self, workloadName, arguments)
# The SkimConfig parameter must be a list of dictionaries where each
# dictionary will have the following keys:
# SkimName
# SkimInput
# SkimSplitAlgo - Optional at workflow creation time
# SkimSplitParams - Optional at workflow creation time
# ConfigCacheID
# Scenario
#
# The ConfigCacheID and Scenaio are mutually exclusive, only one can be
# set. The SkimSplitAlgo and SkimSplitParams don't have to be set when
# the workflow is created but must be set when the workflow is approved.
# The SkimInput is the name of the output module in the processing step
# that will be used as the input for the skim.
self.skimConfigs = arguments.get("SkimConfigs", [])
# These are mostly place holders because the job splitting algo and
# parameters will be updated after the workflow has been created.
self.skimJobSplitAlgo = arguments.get("SkimJobSplitAlgo", "FileBased")
self.skimJobSplitArgs = arguments.get("SkimJobSplitArgs", {
"files_per_job": 1,
"include_parents": True
})
return self.addSkims(workload)
def __call__(self, workloadName, arguments):
"""
_call_
Create a PromptSkimming workload with the given parameters.
"""
configCouchUrl = arguments.get("ConfigCacheUrl",
None) or arguments["CouchURL"]
injectIntoConfigCache(arguments["CMSSWVersion"],
arguments["ScramArch"], arguments["InitCommand"],
arguments["SkimConfig"], workloadName,
configCouchUrl, arguments["CouchDBName"],
arguments.get("EnvPath", None),
arguments.get("BinPath", None))
try:
configCache = ConfigCache(configCouchUrl, arguments["CouchDBName"])
arguments["ConfigCacheID"] = configCache.getIDFromLabel(
workloadName)
if not arguments["ConfigCacheID"]:
logging.error("The configuration was not uploaded to couch")
raise Exception
except Exception:
logging.error(
"There was an exception loading the config out of the")
logging.error(
"ConfigCache. Check the scramOutput.log file in the")
logging.error(
"PromptSkimScheduler directory to find out what went")
logging.error("wrong.")
raise
parsedProcVer = parseT0ProcVer(arguments["ProcessingVersion"],
'PromptSkim')
arguments["ProcessingString"] = parsedProcVer["ProcString"]
arguments["ProcessingVersion"] = parsedProcVer["ProcVer"]
workload = DataProcessingWorkloadFactory.__call__(
self, workloadName, arguments)
# We need to strip off "MSS" as that causes all sorts of problems.
if arguments["CustodialSite"].find("MSS") != -1:
site = arguments["CustodialSite"][:-4]
else:
site = arguments["CustodialSite"]
workload.setSiteWhitelist(site)
workload.setBlockWhitelist(arguments["BlockName"])
return workload
def __call__(self, workloadName, arguments):
"""
_call_
Create a PromptSkimming workload with the given parameters.
"""
self.injectIntoConfigCache(arguments["CMSSWVersion"], arguments["ScramArch"],
arguments["InitCommand"], arguments["SkimConfig"], workloadName,
arguments["CouchURL"], arguments["CouchDBName"])
configCache = ConfigCache(arguments["CouchURL"], arguments["CouchDBName"])
arguments["ProcConfigCacheID"] = configCache.getIDFromLabel(workloadName)
workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments)
workload.setSiteWhitelist(arguments["CustodialSite"])
workload.setBlockWhitelist(arguments["BlockName"])
return workload
def __call__(self, workloadName, arguments):
"""
_call_
Create a PromptSkimming workload with the given parameters.
"""
configCouchUrl = arguments.get("ConfigCacheUrl", None) or arguments["CouchURL"]
injectIntoConfigCache(arguments["CMSSWVersion"], arguments["ScramArch"],
arguments["InitCommand"], arguments["SkimConfig"], workloadName,
configCouchUrl, arguments["CouchDBName"],
arguments.get("EnvPath", None), arguments.get("BinPath", None))
try:
configCache = ConfigCache(configCouchUrl, arguments["CouchDBName"])
arguments["ConfigCacheID"] = configCache.getIDFromLabel(workloadName)
if not arguments["ConfigCacheID"]:
logging.error("The configuration was not uploaded to couch")
raise Exception
except Exception:
logging.error("There was an exception loading the config out of the")
logging.error("ConfigCache. Check the scramOutput.log file in the")
logging.error("PromptSkimScheduler directory to find out what went")
logging.error("wrong.")
raise
parsedProcVer = parseT0ProcVer(arguments["ProcessingVersion"],
'PromptSkim')
arguments["ProcessingString"] = parsedProcVer["ProcString"]
arguments["ProcessingVersion"] = parsedProcVer["ProcVer"]
workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments)
# We need to strip off "MSS" as that causes all sorts of problems.
if arguments["CustodialSite"].find("MSS") != -1:
site = arguments["CustodialSite"][:-4]
else:
site = arguments["CustodialSite"]
workload.setSiteWhitelist(site)
workload.setBlockWhitelist(arguments["BlockName"])
return workload
def __init__(self):
DataProcessingWorkloadFactory.__init__(self)
return
def __init__(self):
DataProcessingWorkloadFactory.__init__(self)
return