def process(self, chunk):
from KaliCalo.MassDistribution import HistoMap
histos = HistoMap()
self.output = histos
for entry in chunk:
efile = entry[0]
enum = entry[1]
histos.read(efile)
print 'Processed %s \t#%d/%d\tEntries:%d\tHistos:%d' % (
efile, enum, self.total, histos.entries(), len(histos))
def SplitHistos():
## first join the filled histograms from all the workers
#print "Splitting histograms"
#import MergeHistos
histos = pt.gethistomaps()
histos.sort()
#hmap = MergeHistos.mergeDBs ( histos )
hmap = HistoMap()
## add all the others
for hm in histos:
hmap.updateFromDB(hm)
print "Adding histograms", hm, hmap.entries()
#hmap.save(pt.histos_location())
## find the last one saved and
## save current as the next one
ih = 1
Pass = 1
while True:
if ih > PassIt / 2:
Pass += 1
ih = 1
if os.path.exists(StoredHistoPath %
(Pass, ih)): ## look through all the existing files
print StoredHistoPath % (Pass, ih), 'exists'
ih += 1
continue
else: ## write to the file next to the latest existing
print 'Saving histograms to ', StoredHistoPath % (Pass, ih)
hmap.save(StoredHistoPath % (Pass, ih))
os.system('chmod a+rwx ' + StoredHistoPath % (Pass, ih))
break
## calculate a number of histograms per worker
nworkers = len(histos)
nhistos = len(hmap.keys())
histos_per_worker = nhistos / nworkers + 1
#print nworkers, nhistos, histos_per_worker
#print "SPLITTING THE HISTOGRAMS BETWEEN ", nworkers, "WORKERS"
#print histos_per_worker, "PER EACH"
data = hmap.split(histos_per_worker)
## get 'All-Ecal' histograms
hA = hmap[AZ]
## inner area
hI = hmap[IZ]
## middle area
hM = hmap[MZ]
## outer area
hO = hmap[OZ]
## To have the global histograms in each histomap
#print len(data)
for dt in data:
if not hA in dt[0]: dt[0].append(hA)
if not hI in dt[0]: dt[0].append(hI)
if not hM in dt[0]: dt[0].append(hM)
if not hO in dt[0]: dt[0].append(hO)
#print len(dt[0]),"HISTOGRAMS"
## rewrite existing databases by the
## parts of the split histo map
num = len(data)
if num < nworkers: data.append(([], 0))
for ih in range(nworkers):
splitmap = HistoMap()
hm = histos[ih]
for jh in data[ih][0]:
splitmap.insert(jh)
print "Clearing previous location", hm
os.remove(hm)
splitmap.save(hm)