コード例 #1
0
    def _splitWork(self,
                   wmspec,
                   parentQueueId=None,
                   data=None,
                   mask=None,
                   team=None):
        """
        Split work from a parent into WorkQeueueElements.

        If data param supplied use that rather than getting input data from
        wmspec. Used for instance when global splits by Block (avoids having to
        modify wmspec block whitelist - thus all appear as same wf in wmbs)

        mask can be used to specify i.e. event range.
        """
        totalUnits = []
        totalToplevelJobs = 0
        totalEvents = 0
        totalLumis = 0
        totalFiles = 0
        # split each top level task into constituent work elements
        # get the acdc server and db name
        for topLevelTask in wmspec.taskIterator():
            spec = getWorkloadFromTask(topLevelTask)
            policyName = spec.startPolicy()
            if not policyName:
                raise RuntimeError(
                    "WMSpec doesn't define policyName, current value: '%s'" %
                    policyName)

            policy = startPolicy(policyName, self.params['SplittingMapping'])
            self.logger.info('Splitting %s with policy %s params = %s' %
                             (topLevelTask.getPathName(), policyName,
                              self.params['SplittingMapping']))
            units = policy(spec, topLevelTask, data, mask)
            for unit in units:
                msg = 'Queuing element %s for %s with %d job(s) split with %s' % (
                    unit.id, unit['Task'].getPathName(), unit['Jobs'],
                    policyName)
                if unit['Inputs']:
                    msg += ' on %s' % unit['Inputs'].keys()[0]
                if unit['Mask']:
                    msg += ' on events %d-%d' % (unit['Mask']['FirstEvent'],
                                                 unit['Mask']['LastEvent'])
                self.logger.info(msg)
                totalToplevelJobs += unit['Jobs']
                totalEvents += unit['NumberOfEvents']
                totalLumis += unit['NumberOfLumis']
                totalFiles += unit['NumberOfFiles']
            totalUnits.extend(units)

        return totalUnits, {
            'total_jobs': totalToplevelJobs,
            'input_events': totalEvents,
            'input_lumis': totalLumis,
            'input_num_files': totalFiles
        }
コード例 #2
0
ファイル: WorkQueue.py プロジェクト: stuartw/WMCore
    def _splitWork(self, wmspec, parentQueueId = None,
                   data = None, mask = None, team = None):
        """
        Split work from a parent into WorkQeueueElements.

        If data param supplied use that rather than getting input data from
        wmspec. Used for instance when global splits by Block (avoids having to
        modify wmspec block whitelist - thus all appear as same wf in wmbs)

        mask can be used to specify i.e. event range.
        """
        totalUnits = []
        totalToplevelJobs = 0
        totalEvents = 0
        totalLumis = 0
        totalFiles = 0
        # split each top level task into constituent work elements
        # get the acdc server and db name
        for topLevelTask in wmspec.taskIterator():
            spec = getWorkloadFromTask(topLevelTask)
            policyName = spec.startPolicy()
            if not policyName:
                raise RuntimeError("WMSpec doesn't define policyName, current value: '%s'" % policyName)

            policy = startPolicy(policyName, self.params['SplittingMapping'])
            self.logger.info('Splitting %s with policy %s params = %s' % (topLevelTask.getPathName(),
                                                policyName, self.params['SplittingMapping']))
            units = policy(spec, topLevelTask, data, mask)
            for unit in units:
                msg = 'Queuing element %s for %s with %d job(s) split with %s' % (unit.id,
                                                unit['Task'].getPathName(), unit['Jobs'], policyName)
                if unit['Inputs']:
                    msg += ' on %s' % unit['Inputs'].keys()[0]
                if unit['Mask']:
                    msg += ' on events %d-%d' % (unit['Mask']['FirstEvent'], unit['Mask']['LastEvent'])
                self.logger.info(msg)
                totalToplevelJobs += unit['Jobs']
                totalEvents += unit['NumberOfEvents']
                totalLumis += unit['NumberOfLumis']
                totalFiles += unit['NumberOfFiles']
            totalUnits.extend(units)

        return totalUnits, {'total_jobs': totalToplevelJobs, 'input_events': totalEvents, 'input_lumis': totalLumis, 'input_num_files': totalFiles}