def __call__(self, workloadName, arguments): """ _call_ Create a ReReco workload with the given parameters. """ workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments) # The SkimConfig parameter must be a list of dictionaries where each # dictionary will have the following keys: # SkimName # SkimInput # SkimSplitAlgo - Optional at workflow creation time # SkimSplitParams - Optional at workflow creation time # ConfigCacheID # Scenario # # The ConfigCacheID and Scenaio are mutually exclusive, only one can be # set. The SkimSplitAlgo and SkimSplitParams don't have to be set when # the workflow is created but must be set when the workflow is approved. # The SkimInput is the name of the output module in the processing step # that will be used as the input for the skim. self.skimConfigs = arguments.get("SkimConfigs", []) # These are mostly place holders because the job splitting algo and # parameters will be updated after the workflow has been created. self.skimJobSplitAlgo = arguments.get("SkimJobSplitAlgo", "FileBased") self.skimJobSplitArgs = arguments.get("SkimJobSplitArgs", {"files_per_job": 1, "include_parents": True}) return self.addSkims(workload)
def __call__(self, workloadName, arguments): """ _call_ Create a ReReco workload with the given parameters. """ workload = DataProcessingWorkloadFactory.__call__( self, workloadName, arguments) # The SkimConfig parameter must be a list of dictionaries where each # dictionary will have the following keys: # SkimName # SkimInput # SkimSplitAlgo - Optional at workflow creation time # SkimSplitParams - Optional at workflow creation time # ConfigCacheID # Scenario # # The ConfigCacheID and Scenaio are mutually exclusive, only one can be # set. The SkimSplitAlgo and SkimSplitParams don't have to be set when # the workflow is created but must be set when the workflow is approved. # The SkimInput is the name of the output module in the processing step # that will be used as the input for the skim. self.skimConfigs = arguments.get("SkimConfigs", []) # These are mostly place holders because the job splitting algo and # parameters will be updated after the workflow has been created. self.skimJobSplitAlgo = arguments.get("SkimJobSplitAlgo", "FileBased") self.skimJobSplitArgs = arguments.get("SkimJobSplitArgs", { "files_per_job": 1, "include_parents": True }) return self.addSkims(workload)
def __call__(self, workloadName, arguments): """ _call_ Create a PromptSkimming workload with the given parameters. """ configCouchUrl = arguments.get("ConfigCacheUrl", None) or arguments["CouchURL"] injectIntoConfigCache(arguments["CMSSWVersion"], arguments["ScramArch"], arguments["InitCommand"], arguments["SkimConfig"], workloadName, configCouchUrl, arguments["CouchDBName"], arguments.get("EnvPath", None), arguments.get("BinPath", None)) try: configCache = ConfigCache(configCouchUrl, arguments["CouchDBName"]) arguments["ConfigCacheID"] = configCache.getIDFromLabel( workloadName) if not arguments["ConfigCacheID"]: logging.error("The configuration was not uploaded to couch") raise Exception except Exception: logging.error( "There was an exception loading the config out of the") logging.error( "ConfigCache. Check the scramOutput.log file in the") logging.error( "PromptSkimScheduler directory to find out what went") logging.error("wrong.") raise parsedProcVer = parseT0ProcVer(arguments["ProcessingVersion"], 'PromptSkim') arguments["ProcessingString"] = parsedProcVer["ProcString"] arguments["ProcessingVersion"] = parsedProcVer["ProcVer"] workload = DataProcessingWorkloadFactory.__call__( self, workloadName, arguments) # We need to strip off "MSS" as that causes all sorts of problems. if arguments["CustodialSite"].find("MSS") != -1: site = arguments["CustodialSite"][:-4] else: site = arguments["CustodialSite"] workload.setSiteWhitelist(site) workload.setBlockWhitelist(arguments["BlockName"]) return workload
def __call__(self, workloadName, arguments): """ _call_ Create a PromptSkimming workload with the given parameters. """ self.injectIntoConfigCache(arguments["CMSSWVersion"], arguments["ScramArch"], arguments["InitCommand"], arguments["SkimConfig"], workloadName, arguments["CouchURL"], arguments["CouchDBName"]) configCache = ConfigCache(arguments["CouchURL"], arguments["CouchDBName"]) arguments["ProcConfigCacheID"] = configCache.getIDFromLabel(workloadName) workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments) workload.setSiteWhitelist(arguments["CustodialSite"]) workload.setBlockWhitelist(arguments["BlockName"]) return workload
def __call__(self, workloadName, arguments): """ _call_ Create a PromptSkimming workload with the given parameters. """ configCouchUrl = arguments.get("ConfigCacheUrl", None) or arguments["CouchURL"] injectIntoConfigCache(arguments["CMSSWVersion"], arguments["ScramArch"], arguments["InitCommand"], arguments["SkimConfig"], workloadName, configCouchUrl, arguments["CouchDBName"], arguments.get("EnvPath", None), arguments.get("BinPath", None)) try: configCache = ConfigCache(configCouchUrl, arguments["CouchDBName"]) arguments["ConfigCacheID"] = configCache.getIDFromLabel(workloadName) if not arguments["ConfigCacheID"]: logging.error("The configuration was not uploaded to couch") raise Exception except Exception: logging.error("There was an exception loading the config out of the") logging.error("ConfigCache. Check the scramOutput.log file in the") logging.error("PromptSkimScheduler directory to find out what went") logging.error("wrong.") raise parsedProcVer = parseT0ProcVer(arguments["ProcessingVersion"], 'PromptSkim') arguments["ProcessingString"] = parsedProcVer["ProcString"] arguments["ProcessingVersion"] = parsedProcVer["ProcVer"] workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments) # We need to strip off "MSS" as that causes all sorts of problems. if arguments["CustodialSite"].find("MSS") != -1: site = arguments["CustodialSite"][:-4] else: site = arguments["CustodialSite"] workload.setSiteWhitelist(site) workload.setBlockWhitelist(arguments["BlockName"]) return workload
def __init__(self): DataProcessingWorkloadFactory.__init__(self) return