Beispiel #1
0
    def __call__(self, workloadName, arguments):
        """
        _call_

        Create a ReReco workload with the given parameters.
        """
        workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments)

        # The SkimConfig parameter must be a list of dictionaries where each
        # dictionary will have the following keys:
        #  SkimName
        #  SkimInput
        #  SkimSplitAlgo - Optional at workflow creation time
        #  SkimSplitParams - Optional at workflow creation time
        #  ConfigCacheID
        #  Scenario
        #
        # The ConfigCacheID and Scenaio are mutually exclusive, only one can be
        # set.  The SkimSplitAlgo and SkimSplitParams don't have to be set when
        # the workflow is created but must be set when the workflow is approved.
        # The SkimInput is the name of the output module in the  processing step
        # that will be used as the input for the skim.
        self.skimConfigs = arguments.get("SkimConfigs", [])

        # These are mostly place holders because the job splitting algo and
        # parameters will be updated after the workflow has been created.
        self.skimJobSplitAlgo = arguments.get("SkimJobSplitAlgo", "FileBased")
        self.skimJobSplitArgs = arguments.get("SkimJobSplitArgs", {"files_per_job": 1, "include_parents": True})

        return self.addSkims(workload)
Beispiel #2
0
    def __call__(self, workloadName, arguments):
        """
        _call_

        Create a ReReco workload with the given parameters.
        """
        workload = DataProcessingWorkloadFactory.__call__(
            self, workloadName, arguments)

        # The SkimConfig parameter must be a list of dictionaries where each
        # dictionary will have the following keys:
        #  SkimName
        #  SkimInput
        #  SkimSplitAlgo - Optional at workflow creation time
        #  SkimSplitParams - Optional at workflow creation time
        #  ConfigCacheID
        #  Scenario
        #
        # The ConfigCacheID and Scenaio are mutually exclusive, only one can be
        # set.  The SkimSplitAlgo and SkimSplitParams don't have to be set when
        # the workflow is created but must be set when the workflow is approved.
        # The SkimInput is the name of the output module in the  processing step
        # that will be used as the input for the skim.
        self.skimConfigs = arguments.get("SkimConfigs", [])

        # These are mostly place holders because the job splitting algo and
        # parameters will be updated after the workflow has been created.
        self.skimJobSplitAlgo = arguments.get("SkimJobSplitAlgo", "FileBased")
        self.skimJobSplitArgs = arguments.get("SkimJobSplitArgs", {
            "files_per_job": 1,
            "include_parents": True
        })

        return self.addSkims(workload)
Beispiel #3
0
    def __call__(self, workloadName, arguments):
        """
        _call_

        Create a PromptSkimming workload with the given parameters.
        """
        configCouchUrl = arguments.get("ConfigCacheUrl",
                                       None) or arguments["CouchURL"]
        injectIntoConfigCache(arguments["CMSSWVersion"],
                              arguments["ScramArch"], arguments["InitCommand"],
                              arguments["SkimConfig"], workloadName,
                              configCouchUrl, arguments["CouchDBName"],
                              arguments.get("EnvPath", None),
                              arguments.get("BinPath", None))

        try:
            configCache = ConfigCache(configCouchUrl, arguments["CouchDBName"])
            arguments["ConfigCacheID"] = configCache.getIDFromLabel(
                workloadName)
            if not arguments["ConfigCacheID"]:
                logging.error("The configuration was not uploaded to couch")
                raise Exception
        except Exception:
            logging.error(
                "There was an exception loading the config out of the")
            logging.error(
                "ConfigCache.  Check the scramOutput.log file in the")
            logging.error(
                "PromptSkimScheduler directory to find out what went")
            logging.error("wrong.")
            raise

        parsedProcVer = parseT0ProcVer(arguments["ProcessingVersion"],
                                       'PromptSkim')
        arguments["ProcessingString"] = parsedProcVer["ProcString"]
        arguments["ProcessingVersion"] = parsedProcVer["ProcVer"]

        workload = DataProcessingWorkloadFactory.__call__(
            self, workloadName, arguments)

        # We need to strip off "MSS" as that causes all sorts of problems.
        if arguments["CustodialSite"].find("MSS") != -1:
            site = arguments["CustodialSite"][:-4]
        else:
            site = arguments["CustodialSite"]

        workload.setSiteWhitelist(site)
        workload.setBlockWhitelist(arguments["BlockName"])
        return workload
Beispiel #4
0
    def __call__(self, workloadName, arguments):
        """
        _call_

        Create a PromptSkimming workload with the given parameters.
        """
        self.injectIntoConfigCache(arguments["CMSSWVersion"], arguments["ScramArch"],
                                   arguments["InitCommand"], arguments["SkimConfig"], workloadName,
                                   arguments["CouchURL"], arguments["CouchDBName"])

        configCache = ConfigCache(arguments["CouchURL"], arguments["CouchDBName"])
        arguments["ProcConfigCacheID"] = configCache.getIDFromLabel(workloadName)
        
        workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments)
        workload.setSiteWhitelist(arguments["CustodialSite"])
        workload.setBlockWhitelist(arguments["BlockName"])
        return workload
Beispiel #5
0
    def __call__(self, workloadName, arguments):
        """
        _call_

        Create a PromptSkimming workload with the given parameters.
        """
        configCouchUrl = arguments.get("ConfigCacheUrl", None) or arguments["CouchURL"]
        injectIntoConfigCache(arguments["CMSSWVersion"], arguments["ScramArch"],
                              arguments["InitCommand"], arguments["SkimConfig"], workloadName,
                              configCouchUrl, arguments["CouchDBName"],
                              arguments.get("EnvPath", None), arguments.get("BinPath", None))

        try:
            configCache = ConfigCache(configCouchUrl, arguments["CouchDBName"])
            arguments["ConfigCacheID"] = configCache.getIDFromLabel(workloadName)
            if not arguments["ConfigCacheID"]:
                logging.error("The configuration was not uploaded to couch")
                raise Exception
        except Exception:
            logging.error("There was an exception loading the config out of the")
            logging.error("ConfigCache.  Check the scramOutput.log file in the")
            logging.error("PromptSkimScheduler directory to find out what went")
            logging.error("wrong.")
            raise

        parsedProcVer = parseT0ProcVer(arguments["ProcessingVersion"],
                                       'PromptSkim')
        arguments["ProcessingString"] = parsedProcVer["ProcString"]
        arguments["ProcessingVersion"] = parsedProcVer["ProcVer"]

        workload = DataProcessingWorkloadFactory.__call__(self, workloadName, arguments)

        # We need to strip off "MSS" as that causes all sorts of problems.
        if arguments["CustodialSite"].find("MSS") != -1:
            site = arguments["CustodialSite"][:-4]
        else:
            site = arguments["CustodialSite"]

        workload.setSiteWhitelist(site)
        workload.setBlockWhitelist(arguments["BlockName"])
        return workload
Beispiel #6
0
 def __init__(self):
     DataProcessingWorkloadFactory.__init__(self)
     return
Beispiel #7
0
 def __init__(self):
     DataProcessingWorkloadFactory.__init__(self)
     return