def manageSecondaries(self,updates,reportAll,reports) : def doUpdate(name) : return updates==True or type(updates)==str and name in updates.split(',') def doReport(name) : return reportAll==True or type(reports)==str and name in reports.split(',') def manage(conf,secondary) : if self.__nocheck and not doUpdate(secondary.name) : return org = organizer(conf['tag'], [ss for ss in self.sampleSpecs(conf['tag']) if ss['name'] in secondary.baseSamples() or not secondary.baseSamples()]) index = next(org.indicesOfStep(secondary.name,secondary.moreNames), next(org.indicesOfStep(secondary.name),None)) if index==None : print " !! Not found: %s %s"%(secondary.name,secondary.moreNames) print org return org.dropSteps( allButIndices = [index]) if doUpdate(secondary.name) : secondary.doCache(org) else : secondary.checkCache(org) def report(conf,secondary) : secondary.allSamples = [ss.weightedName for ss in self.filteredSamples(conf)] if doReport(secondary.name) : secondary.reportCache() confLoopers = [] for conf in self.readyConfs: looper_list = self.listsOfLoopers[conf['tag']] if looper_list: confLoopers.append((conf, looper_list[0])) for _,looper in confLoopers : looper.setupSteps(minimal = True, withBook = False) args = sum([[(conf,secondary) for secondary in filter(self.isSecondary, looper.steps[:self.indexOfInvertedLabel(looper.steps)])] for conf,looper in confLoopers],[]) if reports==True : print '\n'.join(sorted(set(s.name for c,s in args))) sys.exit(0) utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(manage), args) utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(report), args)
def manageSecondaries(self,updates,reports) : def doUpdate(name) : return updates==True or type(updates)==str and name in updates.split(',') def doReport(name) : return reports==True or type(reports)==str and name in reports.split(',') def manage(conf,secondary) : if self.__nocheck and not doUpdate(secondary.name) : return org = organizer(conf['tag'], [ss for ss in self.sampleSpecs(conf['tag']) if ss['name'] in secondary.baseSamples() or not secondary.baseSamples()]) index = next(org.indicesOfStep(secondary.name,secondary.moreNames), next(org.indicesOfStep(secondary.name),None)) if index==None : print " !! Not found: %s %s"%(secondary.name,secondary.moreNames) return org.dropSteps( allButIndices = [index]) if doUpdate(secondary.name) : secondary.doCache(org) else : secondary.checkCache(org) def report(conf,secondary) : if doReport(secondary.name) : secondary.reportCache() confLoopers = [(conf,self.listsOfLoopers[conf['tag']][0]) for conf in self.readyConfs] for _,looper in confLoopers : looper.setupSteps(minimal = True, withBook = False) args = sum([[(conf,secondary) for secondary in filter(self.isSecondary, looper.steps)] for conf,looper in confLoopers],[]) utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(manage), args) utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(report), args)
def manageSecondaries(self,update) : def manage(conf,secondary) : org = self.organizer(conf) index = next(org.indicesOfStep(secondary.name,secondary.moreNames), next(org.indicesOfStep(secondary.name),None)) if index==None : print " !! Not found: %s %s"%(secondary.name,secondary.moreNames) return org.dropSteps( allButIndices = [index]) if update==True or (type(update)==str and secondary.name in update.split(',')) : secondary.doCache(org) else : secondary.checkCache(org) if self.__nocheck : return confLoopers = [(conf,self.listsOfLoopers[conf['tag']][0]) for conf in self.readyConfs] for _,looper in confLoopers : looper.setupSteps(minimal = True, withBook = False) args = sum([[(conf,secondary) for secondary in filter(self.isSecondary, looper.steps)] for conf,looper in confLoopers],[]) utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(manage), args)
def mergeAllOutput(self) : args = sum([[(conf['tag'],looper) for looper in self.listsOfLoopers[conf['tag']]] for conf in self.configurations],[]) utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(self.mergeOutput), args)
def concludeAll(self) : utils.operateOnListUsingQueue( configuration.nCoresDefault(), utils.qWorker(self.conclude), zip(self.readyConfs) ) def organizer(self, config) : return organizer.organizer(config['tag'], self.sampleSpecs(config['tag']))
def recordedInvMicrobarns(json) : jsonALT = dict([(int(run),sum([range(begin,end+1) for begin,end in lumis],[])) for run,lumis in json.items()]) session = sessionManager.sessionManager("frontier://LumiCalc/CMS_LUMI_PROD").openSession( cpp2sqltype = [('unsigned int','NUMBER(10)'),('unsigned long long','NUMBER(20)')]) session.transaction().start(True) lumidata = lumiCalcAPI.lumiForRange( session.nominalSchema(), jsonALT, norm = 1.0, finecorrections = lumiCorrections.pixelcorrectionsForRange(session.nominalSchema(),jsonALT.keys()), lumitype='PIXEL', branchName='DATA') return sum( sum(data[6] for data in lumis) for run,lumis in lumidata.iteritems() if lumis) def recordedInvMicrobarnsShotgun(jsons, cores = 2, cacheDir = './' ) : pickles = ["%s/%d.pickle"%(cacheDir,hash(str(sorted([(key,val) for key,val in json.iteritems()])))) for json in jsons] def worker(pickle, json) : if not os.path.exists(pickle) : utils.writePickle(pickle, recordedInvMicrobarns(json)) utils.operateOnListUsingQueue(cores, utils.qWorker(worker), zip(pickles,jsons)) return [utils.readPickle(pickle) for pickle in pickles ] if __name__=='__main__' : print if len(sys.argv)<2 : print 'Pass list of "{json}" and/or filenames as argument'; sys.exit(0) def output(arg) : json = eval(arg if '{' in arg else open(arg).readline()) lumi = recordedInvMicrobarns(json) print "%.4f/pb in %s"%(lumi/1e6,arg) print utils.operateOnListUsingQueue(configuration.nCoresDefault(), utils.qWorker(output), [(a,) for a in sys.argv[1:]])
def concludeAll(self) : utils.operateOnListUsingQueue( configuration.nCoresDefault(), utils.qWorker(self.conclude), zip(self.readyConfs) ) def organizer(self, config, verbose = True, prefixesNoScale=[]) :