示例#1
0
    def manageSecondaries(self,updates,reportAll,reports) :
        def doUpdate(name) : return updates==True or type(updates)==str and name in updates.split(',')
        def doReport(name) : return reportAll==True or type(reports)==str and name in reports.split(',')

        def manage(conf,secondary) :
            if self.__nocheck and not doUpdate(secondary.name) : return
            org = organizer(conf['tag'], [ss for ss in self.sampleSpecs(conf['tag'])
                                          if ss['name'] in secondary.baseSamples() or not secondary.baseSamples()])
            index = next(org.indicesOfStep(secondary.name,secondary.moreNames), next(org.indicesOfStep(secondary.name),None))
            if index==None :
                print " !! Not found: %s    %s"%(secondary.name,secondary.moreNames)
                print org
                return
            org.dropSteps( allButIndices = [index])
            if doUpdate(secondary.name) : secondary.doCache(org)
            else : secondary.checkCache(org)

        def report(conf,secondary) :
            secondary.allSamples = [ss.weightedName for ss in self.filteredSamples(conf)]
            if doReport(secondary.name) : secondary.reportCache()
            
        confLoopers = []
        for conf in self.readyConfs:
            looper_list = self.listsOfLoopers[conf['tag']]
            if looper_list:
                confLoopers.append((conf, looper_list[0]))
        for _,looper in confLoopers : looper.setupSteps(minimal = True, withBook = False)
        args = sum([[(conf,secondary) for secondary in filter(self.isSecondary, looper.steps[:self.indexOfInvertedLabel(looper.steps)])] for conf,looper in confLoopers],[])
        if reports==True :
            print '\n'.join(sorted(set(s.name for c,s in args)))
            sys.exit(0)
        utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(manage), args)
        utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(report), args)
示例#2
0
 def loop(self) :
     wl = self.workList()
     
     if self.__jobId!=None : self.func(*(wl[0]))
     elif not self.__profile : utils.operateOnListUsingQueue(self.__loop, utils.qWorker(self.func), wl)
     else :
         import cProfile
         self.listOfLoopersForProf = wl
         cProfile.run("someInstance.profileLoop()","resultProfile.out")
示例#3
0
    def makeInputFileLists(self) :
        def makeFileList(name) :
            if os.path.exists(self.inputFilesListFile(name)) and self.useCachedFileLists() : return
            fileNames = eval(self.sampleDict[name].filesCommand)
            assert fileNames, "The command '%s' produced an empty list of files"%self.sampleDict[name].filesCommand
            tmpDir,localFileName,globalFileName = self.globalToLocal(self.inputFilesListFile(name))
            utils.writePickle(localFileName, fileNames)
            self.localToGlobal(tmpDir, localFileName, globalFileName)

        sampleNames = set(sum([[sampleSpec.name for sampleSpec in self.filteredSamples(conf)] for conf in self.configurations],[]))
        utils.operateOnListUsingQueue(self.__loop, utils.qWorker(makeFileList), [(name,) for name in sampleNames] )
示例#4
0
    def makeInputFileLists(self) :
        def nEventsFile(fileName):
            if not configuration.computeEntriesAtMakeFileList(): return None
            return utils.nEventsFile(fileName,'/'.join(self.mainTree()))
        def makeFileList(name) :
            fName = self.inputFilesListFile(name)
            if os.path.exists(fName) and self.useCachedFileLists() : return
            fileNames = eval(self.sampleDict[name].filesCommand)
            assert fileNames, "The command '%s' produced an empty list of files"%self.sampleDict[name].filesCommand
            utils.mkdir(os.path.dirname(fName))
            utils.writePickle(fName, zip(fileNames,map(nEventsFile, fileNames)))

        sampleNames = set(sum([[sampleSpec.name for sampleSpec in self.filteredSamples(conf)] for conf in self.configurations],[]))
        utils.operateOnListUsingQueue(self.__loop, utils.qWorker(makeFileList), [(name,) for name in sampleNames] )
示例#5
0
文件: analysis.py 项目: aharel/supy
    def manageSecondaries(self,update) :
        def manage(conf,secondary) :
            org = self.organizer(conf)
            index = next(org.indicesOfStep(secondary.name,secondary.moreNames), next(org.indicesOfStep(secondary.name),None))
            if index==None :
                print " !! Not found: %s    %s"%(secondary.name,secondary.moreNames)
                return
            org.dropSteps( allButIndices = [index])
            if update==True or (type(update)==str and secondary.name in update.split(',')) :
                secondary.doCache(org)
            else : secondary.checkCache(org)

        if self.__nocheck : return
        confLoopers = [(conf,self.listsOfLoopers[conf['tag']][0]) for conf in self.readyConfs]
        for _,looper in confLoopers : looper.setupSteps(minimal = True, withBook = False)
        args = sum([[(conf,secondary) for secondary in filter(self.isSecondary, looper.steps)] for conf,looper in confLoopers],[])
        utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(manage), args)
示例#6
0
文件: batch.py 项目: gerbaudo/statsTA
    def __init__(self,cmds, nCores=4, site=None):
        self.nCores = nCores
        self.cmds = cmds

        if site == 'ic':
            localdir =  '/'.join(__file__.split('/')[:-1])
            prelude = '''
export SCRAM_ARCH=slc5_amd64_gcc472
source /vols/cms/grid/setup.sh
cd /home/hep/bbetchar/work/CMSSW_6_2_0_patch1/src
eval `scram runtime -sh`
cd %s
'''%localdir
            def sub(iJob,cmd):
                with tempfile.NamedTemporaryFile('w',prefix='job%04d_'%iJob,suffix='.sh') as tmp:
                    print>>tmp, prelude
                    print>>tmp, cmd
                    tmp.flush()
                    os.system('./ic_cmsSub.sh %s'%tmp.name)
            utils.operateOnListUsingQueue(self.nCores, utils.qWorker(sub), enumerate(self.cmds))
        else:
            self.default()
示例#7
0
    def manageSecondaries(self,updates,reports) :
        def doUpdate(name) : return updates==True or type(updates)==str and name in updates.split(',')
        def doReport(name) : return reports==True or type(reports)==str and name in reports.split(',')

        def manage(conf,secondary) :
            if self.__nocheck and not doUpdate(secondary.name) : return
            org = organizer(conf['tag'], [ss for ss in self.sampleSpecs(conf['tag'])
                                          if ss['name'] in secondary.baseSamples() or not secondary.baseSamples()])
            index = next(org.indicesOfStep(secondary.name,secondary.moreNames), next(org.indicesOfStep(secondary.name),None))
            if index==None :
                print " !! Not found: %s    %s"%(secondary.name,secondary.moreNames)
                return
            org.dropSteps( allButIndices = [index])
            if doUpdate(secondary.name) : secondary.doCache(org)
            else : secondary.checkCache(org)

        def report(conf,secondary) :
            if doReport(secondary.name) : secondary.reportCache()
            
        confLoopers = [(conf,self.listsOfLoopers[conf['tag']][0]) for conf in self.readyConfs]
        for _,looper in confLoopers : looper.setupSteps(minimal = True, withBook = False)
        args = sum([[(conf,secondary) for secondary in filter(self.isSecondary, looper.steps)] for conf,looper in confLoopers],[])
        utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(manage), args)
        utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(report), args)
示例#8
0
 def mergeAllOutput(self) :
     args = sum([[(conf['tag'],looper) for looper in self.listsOfLoopers[conf['tag']]] for conf in self.configurations],[])
     utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(self.mergeOutput), args)
示例#9
0
 def concludeAll(self) : utils.operateOnListUsingQueue( configuration.nCoresDefault(), utils.qWorker(self.conclude), zip(self.readyConfs) )
 def organizer(self, config) : return organizer.organizer(config['tag'], self.sampleSpecs(config['tag']))
示例#10
0
 def concludeAll(self) : utils.operateOnListUsingQueue( configuration.nCoresDefault(), utils.qWorker(self.conclude), zip(self.readyConfs) )
 def organizer(self, config, verbose = True, prefixesNoScale=[]) :
示例#11
0
文件: batch.py 项目: gerbaudo/statsTA
 def default(self):
     utils.operateOnListUsingQueue(self.nCores, utils.qWorker(os.system), zip(self.cmds))
示例#12
0
文件: luminosity.py 项目: aharel/supy
def recordedInvMicrobarnsShotgun(jsons, cores = 2, cacheDir = './' ) :
    pickles = ["%s/%d.pickle"%(cacheDir,hash(str(sorted([(key,val) for key,val in json.iteritems()])))) for json in jsons]
    def worker(pickle, json) :
        if not os.path.exists(pickle) : utils.writePickle(pickle, recordedInvMicrobarns(json))
    utils.operateOnListUsingQueue(cores, utils.qWorker(worker), zip(pickles,jsons))
    return [utils.readPickle(pickle) for pickle in pickles ]
示例#13
0
文件: luminosity.py 项目: aharel/supy
def recordedInvMicrobarns(json) :
    pars = lumiQueryAPI.ParametersObject()
    pars.noWarnings  = True
    pars.norm        = 1.0
    pars.lumiversion = '0001'
    pars.beammode    = 'STABLE BEAMS'

    session,svc =  lumiQueryAPI.setupSession( connectString = 'frontier://LumiCalc/CMS_LUMI_PROD',
                                              parameters = pars, siteconfpath = None, debug = False)
    lumidata =  lumiQueryAPI.recordedLumiForRange (session, pars, jsonToIFP(json))    
    return sum(lumiQueryAPI.calculateTotalRecorded(dataperRun[2]) for dataperRun in lumidata if dataperRun[1])

def recordedInvMicrobarnsShotgun(jsons, cores = 2, cacheDir = './' ) :
    pickles = ["%s/%d.pickle"%(cacheDir,hash(str(sorted([(key,val) for key,val in json.iteritems()])))) for json in jsons]
    def worker(pickle, json) :
        if not os.path.exists(pickle) : utils.writePickle(pickle, recordedInvMicrobarns(json))
    utils.operateOnListUsingQueue(cores, utils.qWorker(worker), zip(pickles,jsons))
    return [utils.readPickle(pickle) for pickle in pickles ]

if __name__=='__main__' :
    print
    import utils
    if len(sys.argv)<2 : print 'Pass list of "{json}" and/or filenames as argument'; sys.exit(0)

    def output(arg) :
        json = eval(arg if '{' in arg else open(arg).readline())
        lumi = recordedInvMicrobarns(json)
        print "%.4f/pb in %s"%(lumi/1e6,arg)
        print
    utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(output), [(a,) for a in sys.argv[1:]])