def makeSamples(SIZE, HOLES):
with Numpy() as numpy:
if numpy is None:
return {"empty": None, "positive": None, "boolean": None, "noholes": None, "withholes": None, "withholes2": None}
empty = numpy.array([], dtype=float)
if numpy is not None:
rand = random.Random(12345)
positive = numpy.array([abs(rand.gauss(0, 1)) + 1e-12 for i in xrange(SIZE)])
assert all(x > 0.0 for x in positive)
boolean = positive > 1.5
noholes = numpy.array([rand.gauss(0, 1) for i in xrange(SIZE)])
withholes = numpy.array([rand.gauss(0, 1) for i in xrange(SIZE)])
for i in xrange(HOLES):
withholes[rand.randint(0, SIZE)] = float("nan")
for i in xrange(HOLES):
withholes[rand.randint(0, SIZE)] = float("inf")
for i in xrange(HOLES):
withholes[rand.randint(0, SIZE)] = float("-inf")
withholes2 = numpy.array([rand.gauss(0, 1) for i in xrange(SIZE)])
for i in xrange(HOLES):
withholes2[rand.randint(0, SIZE)] = float("nan")
for i in xrange(HOLES):
withholes2[rand.randint(0, SIZE)] = float("inf")
for i in xrange(HOLES):
withholes2[rand.randint(0, SIZE)] = float("-inf")
return {"empty": empty, "positive": positive, "boolean": boolean, "noholes": noholes, "withholes": withholes, "withholes2": withholes2}
def plotroot(self, name, title="", binType="D"):
import ROOT
constructor = getattr(ROOT, "TH2" + binType)
sample = self.values[0]
th2 = constructor(name, title, int(self.num), float(self.low),
float(self.high), int(sample.num), float(sample.low),
float(sample.high))
for i in xrange(self.num):
for j in xrange(sample.num):
th2.SetBinContent(i + 1, j + 1,
self.values[i].values[j].entries)
return th2
def bin_entries(self, low=None, high=None, xvalues=[]):
"""
Returns bin values
Possible to set range with low and high params, and list of selected x-values
:param low: lower edge of range, default is None
:param high: higher edge of range, default is None
:param xvalues: list of x-values to get entries of, alternative to low and high
:returns: numpy array with numbers of entries for selected bins
:rtype: numpy.array
"""
import numpy as np
# trivial case
if low is None and high is None and len(xvalues) == 0:
return np.array([b[1].entries for b in self.bins])
# catch weird cases
elif low is not None and high is not None and len(xvalues) == 0:
if low > high:
raise RuntimeError('low {low} greater than high {high}'.format(
low=low, high=high))
# entries at request list of x-values
elif len(xvalues) > 0:
return np.array([(self.bins[self.index(x)])[1].entries
for x in xvalues])
# lowest, highest edge reset
if low is None:
low = float("-inf")
if high is None:
high = float("inf")
# return bin entries
lidx = self._lower_index(low)
hidx = self._upper_index(high)
return np.array([(self.bins[i])[1].entries
for i in xrange(lidx, hidx + 1)])
def plotroot(self, name, title=""):
import ROOT
if self.minBin is None or self.maxBin is None:
tprofile = ROOT.TProfile(name, title, 1, self.origin,
self.origin + 1.0)
else:
tprofile = ROOT.TProfile(name, title,
1 + self.maxBin - self.minBin, self.low,
self.high)
for i, index in enumerate(xrange(self.minBin, self.maxBin + 1)):
if index in self.bins:
v = self.bins[index]
if not math.isnan(v.mean):
tprofile.SetBinError(
i + 1,
math.sqrt(v.entries *
(v.variance + v.mean * v.mean)))
tprofile.SetBinContent(i + 1, v.entries * v.mean)
tprofile.SetBinEntries(i + 1, v.entries)
tprofile.SetBinContent(0, 0.0)
tprofile.SetBinEntries(0, 0.0)
tprofile.SetBinContent(1 + self.maxBin - self.minBin, 0.0)
tprofile.SetBinEntries(1 + self.maxBin - self.minBin, 0.0)
tprofile.SetEntries(self.entries)
return tprofile
def testMaximize(self):
for i in xrange(11):
left, right = self.simple[:i], self.simple[i:]
leftMaximizing = Maximize(named("something", lambda x: x))
rightMaximizing = Maximize(named("something", lambda x: x))
for _ in left:
leftMaximizing.fill(_)
for _ in right:
rightMaximizing.fill(_)
if len(left) > 0:
self.assertAlmostEqual(leftMaximizing.max, max(left))
else:
self.assertTrue(math.isnan(leftMaximizing.max))
if len(right) > 0:
self.assertAlmostEqual(rightMaximizing.max, max(right))
else:
self.assertTrue(math.isnan(rightMaximizing.max))
finalResult = leftMaximizing + rightMaximizing
self.assertAlmostEqual(finalResult.max, max(self.simple))
self.checkScaling(leftMaximizing)
self.checkScaling(leftMaximizing.toImmutable())
self.checkJson(leftMaximizing)
self.checkPickle(leftMaximizing)
self.checkName(leftMaximizing)
def plotmatplotlib(self, name=None, **kwargs):
""" Plotting method for SparselyBin of Average
name : title of the plot.
kwargs : matplotlib.collections.LineCollection properties.
Returns a matplotlib.axes instance
"""
import matplotlib.pyplot as plt
import numpy as np
ax = plt.gca()
xmins = np.arange(self.low, self.high, self.binWidth)
xmaxs = np.arange(self.low + self.binWidth, self.high + self.binWidth, self.binWidth)
means = np.nan*np.ones(xmaxs.shape)
for i in xrange(self.minBin, self.maxBin + 1):
if i in self.bins:
means[i - self.minBin] = self.bins[i].mean
idx = np.isfinite(means)
ax.hlines(means[idx], xmins[idx], xmaxs[idx], **kwargs)
if name is not None:
ax.set_title(name)
else:
ax.set_title(self.name)
return ax
def testSumWithWeightingFactorStringFunctions(self):
for i in xrange(11):
left, right = self.struct[:i], self.struct[i:]
leftSumming = Select("int", Sum("double * 2"))
rightSumming = Select("int", Sum("double * 2"))
for _ in left:
leftSumming.fill(_)
for _ in right:
rightSumming.fill(_)
self.assertAlmostEqual(
leftSumming.cut.sum,
sum(_.double * 2 * _.int for _ in left if _.int > 0))
self.assertAlmostEqual(
rightSumming.cut.sum,
sum(_.double * 2 * _.int for _ in right if _.int > 0))
finalResult = leftSumming + rightSumming
self.assertAlmostEqual(
finalResult.cut.sum,
sum(_.double * 2 * _.int for _ in self.struct if _.int > 0))
self.checkScaling(leftSumming)
self.checkScaling(leftSumming.toImmutable())
self.checkJson(leftSumming)
self.checkPickle(leftSumming)
self.checkName(leftSumming)
def testSumWithFilterStringFunctions(self):
for i in xrange(11):
left, right = self.struct[:i], self.struct[i:]
leftSumming = Select("not bool", Sum("double + 1"))
rightSumming = Select("not bool", Sum("double + 1"))
for _ in left:
leftSumming.fill(_)
for _ in right:
rightSumming.fill(_)
self.assertAlmostEqual(
leftSumming.cut.sum,
sum(_.double + 1 for _ in left if not _.bool))
self.assertAlmostEqual(
rightSumming.cut.sum,
sum(_.double + 1 for _ in right if not _.bool))
finalResult = leftSumming + rightSumming
self.assertAlmostEqual(
finalResult.cut.sum,
sum(_.double + 1 for _ in self.struct if not _.bool))
self.checkScaling(leftSumming)
self.checkScaling(leftSumming.toImmutable())
self.checkJson(leftSumming)
self.checkPickle(leftSumming)
self.checkName(leftSumming)
def testSumStringFunctions(self):
for i in xrange(11):
left, right = self.simple[:i], self.simple[i:]
leftSumming = Sum("_ + 1")
rightSumming = Sum("datum + 1")
for _ in left:
leftSumming.fill(_)
for _ in right:
rightSumming.fill(_)
self.assertAlmostEqual(leftSumming.sum, sum(left) + len(left))
self.assertAlmostEqual(rightSumming.sum, sum(right) + len(right))
finalResult = leftSumming + rightSumming
self.assertAlmostEqual(finalResult.sum,
sum(self.simple) + len(self.simple))
self.checkScaling(leftSumming)
self.checkScaling(leftSumming.toImmutable())
self.checkJson(leftSumming)
self.checkPickle(leftSumming)
self.checkName(leftSumming)
def testSumWithFilter(self):
for i in xrange(11):
left, right = self.struct[:i], self.struct[i:]
leftSumming = Select(lambda x: x.bool, Sum(lambda x: x.double))
rightSumming = Select(lambda x: x.bool, Sum(lambda x: x.double))
for _ in left:
leftSumming.fill(_)
for _ in right:
rightSumming.fill(_)
self.assertAlmostEqual(leftSumming.cut.sum,
sum(_.double for _ in left if _.bool))
self.assertAlmostEqual(rightSumming.cut.sum,
sum(_.double for _ in right if _.bool))
finalResult = leftSumming + rightSumming
self.assertAlmostEqual(
finalResult.cut.sum,
sum(_.double for _ in self.struct if _.bool))
self.checkScaling(leftSumming)
self.checkScaling(leftSumming.toImmutable())
self.checkJson(leftSumming)
self.checkPickle(leftSumming)
self.checkName(leftSumming)
def testCountWithFilter(self):
for i in xrange(11):
left, right = self.simple[:i], self.simple[i:]
leftCounting = Select(named("something", lambda x: x > 0.0),
Count())
rightCounting = Select(named("something", lambda x: x > 0.0),
Count())
for _ in left:
leftCounting.fill(_)
for _ in right:
rightCounting.fill(_)
self.assertEqual(leftCounting.cut.entries,
len(list(filter(lambda x: x > 0.0, left))))
self.assertEqual(rightCounting.cut.entries,
len(list(filter(lambda x: x > 0.0, right))))
finalResult = leftCounting + rightCounting
self.assertEqual(finalResult.cut.entries,
len(list(filter(lambda x: x > 0.0, self.simple))))
self.checkScaling(leftCounting)
self.checkScaling(leftCounting.toImmutable())
self.checkJson(leftCounting)
self.checkPickle(leftCounting)
self.checkName(leftCounting)
def plotmatplotlib(self, name=None, aspect=True, **kwargs):
""" Plotting method for
"""
import matplotlib.pyplot as plt
import numpy as np
ax = plt.gca()
xmins = np.arange(self.low, self.high, self.binWidth)
xmaxs = np.arange(self.low + self.binWidth, self.high + self.binWidth, self.binWidth)
means = np.nan*np.ones(xmaxs.shape)
variances = np.nan*np.ones(xmaxs.shape)
counts = np.nan*np.ones(xmaxs.shape)
for i in xrange(self.minBin, self.maxBin + 1):
if i in self.bins:
means[i - self.minBin] = self.bins[i].mean
variances[i - self.minBin] = self.bins[i].variance
counts[i - self.minBin] = self.bins[i].entries
idx = np.isfinite(means)
# pull out non nans
means = means[idx]
xmins = xmins[idx]
xmaxs = xmaxs[idx]
variances = variances[idx]
ax.hlines(means, xmins, xmaxs, **kwargs)
bin_centers = (self.binWidth/2.0) + xmins
if aspect is True:
ymins = [means[i] - np.sqrt(variances[i])/np.sqrt(counts[i]) for i in xrange(len(means))]
ymaxs = [means[i] + np.sqrt(variances[i])/np.sqrt(counts[i]) for i in xrange(len(means))]
else:
ymins = [means[i] - np.sqrt(variances[i]) for i in xrange(len(means))]
ymaxs = [means[i] + np.sqrt(variances[i]) for i in xrange(len(means))]
ax.vlines(bin_centers, ymins, ymaxs, **kwargs)
if name is not None:
ax.set_title(name)
else:
ax.set_title(self.name)
return ax
def _clingUpdate(self, filler, *extractorPrefix):
obj = self._clingExpandPrefix(filler, *extractorPrefix)
self.entries += obj.entries
for i in obj.values:
key = i.first
if self.range[0] == "N" and len(self.range) > 1:
key = tuple(getattr(key, "v" + str(x)) for x in xrange(self.dimension))
if key not in self.values:
self.values[key] = 0.0
self.values[key] += i.second
def testDeviateWithWeightingFactor(self):
for i in xrange(11):
left, right = self.struct[:i], self.struct[i:]
leftDeviating = Select(lambda x: x.int,
Deviate(lambda x: x.double))
rightDeviating = Select(lambda x: x.int,
Deviate(lambda x: x.double))
for _ in left:
leftDeviating.fill(_)
for _ in right:
rightDeviating.fill(_)
if sum(map(lambda _: _.int if _.int > 0.0 else 0.0, left)) == 0.0:
self.assertTrue(math.isnan(leftDeviating.cut.mean))
self.assertTrue(math.isnan(leftDeviating.cut.variance))
else:
self.assertAlmostEqual(
leftDeviating.cut.mean,
self.meanWeighted(list(map(lambda _: _.double, left)),
list(map(lambda _: _.int, left))))
self.assertAlmostEqual(
leftDeviating.cut.variance,
self.varianceWeighted(list(map(lambda _: _.double, left)),
list(map(lambda _: _.int, left))))
if sum(map(lambda _: _.int if _.int > 0.0 else 0.0, right)) == 0.0:
self.assertTrue(math.isnan(rightDeviating.cut.mean))
self.assertTrue(math.isnan(rightDeviating.cut.variance))
else:
self.assertAlmostEqual(
rightDeviating.cut.mean,
self.meanWeighted(list(map(lambda _: _.double, right)),
list(map(lambda _: _.int, right))))
self.assertAlmostEqual(
rightDeviating.cut.variance,
self.varianceWeighted(list(map(lambda _: _.double, right)),
list(map(lambda _: _.int, right))))
finalResult = leftDeviating + rightDeviating
self.assertAlmostEqual(
finalResult.cut.variance,
self.varianceWeighted(
list(map(lambda _: _.double, self.struct)),
list(map(lambda _: _.int, self.struct))))
self.checkScaling(leftDeviating)
self.checkScaling(leftDeviating.toImmutable())
self.checkJson(leftDeviating)
self.checkPickle(leftDeviating)
self.checkName(leftDeviating)
def index(self, x, greater=True):
"""Find the closest index to ``x``."""
for index in xrange(len(self.bins)):
if index == len(self.bins) - 1:
return index
thisCenter = self.bins[index][0]
nextCenter = self.bins[index + 1][0]
if greater:
if x < (thisCenter + nextCenter) / 2.0:
return index
else:
if x <= (thisCenter + nextCenter) / 2.0:
return index
def plotroot(self, name, title="", binType="D"):
import ROOT
constructor = getattr(ROOT, "TH1" + binType)
if self.minBin is None or self.maxBin is None:
th1 = constructor(name, title, 1, self.origin, self.origin + 1.0)
else:
size = 1 + self.maxBin - self.minBin
th1 = constructor(name, title, size, self.low, self.high)
setTH1(self.entries, [
self.bins[i].entries if i in self.bins else 0.0
for i in xrange(self.minBin, self.maxBin + 1)
], 0.0, 0.0, th1)
return th1
def build(*ys):
"""Create a Stack out of pre-existing containers, which might have been aggregated on different streams.
Parameters:
aggregators (list of :doc:`Container <histogrammar.defs.Container>`): this function will attempt to add
them, so they must also have the same binning/bounds/etc.
"""
from functools import reduce
if not all(isinstance(y, Container) for y in ys):
raise TypeError("ys must all be Containers")
entries = sum(y.entries for y in ys)
bins = []
for i in xrange(len(ys)):
bins.append((float("nan"), reduce(lambda a, b: a + b, ys[i:])))
return Stack.ed(entries, bins, Count.ed(0.0))
def testDeviateWithFilter(self):
for i in xrange(11):
left, right = self.struct[:i], self.struct[i:]
leftDeviating = Select(lambda x: x.bool,
Deviate(lambda x: x.double))
rightDeviating = Select(lambda x: x.bool,
Deviate(lambda x: x.double))
for _ in left:
leftDeviating.fill(_)
for _ in right:
rightDeviating.fill(_)
if len([_.double for _ in left if _.bool]) == 0:
self.assertTrue(math.isnan(leftDeviating.cut.mean))
self.assertTrue(math.isnan(leftDeviating.cut.variance))
else:
self.assertAlmostEqual(
leftDeviating.cut.mean,
self.mean([_.double for _ in left if _.bool]))
self.assertAlmostEqual(
leftDeviating.cut.variance,
self.variance([_.double for _ in left if _.bool]))
if len([_.double for _ in right if _.bool]) == 0:
self.assertTrue(math.isnan(rightDeviating.cut.mean))
self.assertTrue(math.isnan(rightDeviating.cut.variance))
else:
self.assertAlmostEqual(
rightDeviating.cut.mean,
self.mean([_.double for _ in right if _.bool]))
self.assertAlmostEqual(
rightDeviating.cut.variance,
self.variance([_.double for _ in right if _.bool]))
finalResult = leftDeviating + rightDeviating
self.assertAlmostEqual(
finalResult.cut.variance,
self.variance([_.double for _ in self.struct if _.bool]))
self.checkScaling(leftDeviating)
self.checkScaling(leftDeviating.toImmutable())
self.checkJson(leftDeviating)
self.checkPickle(leftDeviating)
self.checkName(leftDeviating)
def plotmatplotlib(self, name=None, **kwargs):
""" Plotting method for
"""
import matplotlib.pyplot as plt
import numpy as np
ax = plt.gca()
if isinstance(self.numerator, HistogramMethods):
fracs = [x[0].entries / float(x[1].entries) for x in zip(self.numerator.values, self.denominator.values)]
xranges = [self.numerator.range(x) for x in self.numerator.indexes]
xmins = [x[0] for x in xranges]
xmaxs = [x[1] for x in xranges]
ax.hlines(fracs, xmins, xmaxs, **kwargs)
elif isinstance(self.numerator, SparselyHistogramMethods):
assert self.numerator.binWidth == self.denominator.binWidth,\
"Fraction numerator and denominator histograms must have same binWidth."
numerator = self.numerator
denominator = self.denominator
xmins = np.arange(numerator.low, numerator.high, numerator.binWidth)
xmaxs = np.arange(
numerator.low +
numerator.binWidth,
numerator.high +
numerator.binWidth,
numerator.binWidth)
fracs = np.nan*np.zeros(xmaxs.shape)
for i in xrange(denominator.minBin, denominator.maxBin + 1):
if i in self.numerator.bins and i in self.denominator.bins:
fracs[i - denominator.minBin] = numerator.bins[i].entries / denominator.bins[i].entries
idx = np.isfinite(fracs)
ax.hlines(fracs[idx], xmins[idx], xmaxs[idx], **kwargs)
ax.set_ylim((0.0, 1.0))
if name is not None:
ax.set_title(name)
else:
ax.set_title(self.name)
return ax
def plotroot(self, numeratorName, denominatorName):
import ROOT
denominator = self.denominator.plotroot(denominatorName)
num = denominator.GetNbinsX()
low = denominator.GetBinLowEdge(1)
high = denominator.GetBinLowEdge(num) + denominator.GetBinWidth(num)
numerator = ROOT.TH1D(numeratorName, "", num, low, high)
if isinstance(self.numerator, HistogramMethods):
setTH1(self.numerator.entries,
[x.entries for x in self.numerator.values],
self.numerator.underflow.entries,
self.numerator.overflow.entries, numerator)
elif isinstance(self.numerator, SparselyHistogramMethods):
setTH1(self.numerator.entries, [
self.numerator.bins[i].entries
if i in self.numerator.bins else 0.0 for i in xrange(
self.denominator.minBin, self.denominator.maxBin + 1)
], 0.0, 0.0, numerator)
return ROOT.TEfficiency(numerator, denominator)
def testDeviate(self):
for i in xrange(11):
left, right = self.simple[:i], self.simple[i:]
leftDeviating = Deviate(named("something", lambda x: x))
rightDeviating = Deviate(named("something", lambda x: x))
for _ in left:
leftDeviating.fill(_)
for _ in right:
rightDeviating.fill(_)
if len(left) == 0:
self.assertTrue(math.isnan(leftDeviating.mean))
self.assertTrue(math.isnan(leftDeviating.variance))
else:
self.assertAlmostEqual(leftDeviating.mean, self.mean(left))
self.assertAlmostEqual(leftDeviating.variance,
self.variance(left))
if len(right) == 0:
self.assertTrue(math.isnan(rightDeviating.mean))
self.assertTrue(math.isnan(rightDeviating.variance))
else:
self.assertAlmostEqual(rightDeviating.mean, self.mean(right))
self.assertAlmostEqual(rightDeviating.variance,
self.variance(right))
finalResult = leftDeviating + rightDeviating
self.assertAlmostEqual(finalResult.variance,
self.variance(self.simple))
self.checkScaling(leftDeviating)
self.checkScaling(leftDeviating.toImmutable())
self.checkJson(leftDeviating)
self.checkPickle(leftDeviating)
self.checkName(leftDeviating)
def testSum(self):
for i in xrange(11):
left, right = self.simple[:i], self.simple[i:]
leftSumming = Sum(named("something", lambda x: x))
rightSumming = Sum(named("something", lambda x: x))
for _ in left:
leftSumming.fill(_)
for _ in right:
rightSumming.fill(_)
self.assertAlmostEqual(leftSumming.sum, sum(left))
self.assertAlmostEqual(rightSumming.sum, sum(right))
finalResult = leftSumming + rightSumming
self.assertAlmostEqual(finalResult.sum, sum(self.simple))
self.checkScaling(leftSumming)
self.checkScaling(leftSumming.toImmutable())
self.checkJson(leftSumming)
self.checkPickle(leftSumming)
self.checkName(leftSumming)
def testCount(self):
for i in xrange(11):
left, right = self.simple[:i], self.simple[i:]
leftCounting = Count()
rightCounting = Count()
for _ in left:
leftCounting.fill(_)
for _ in right:
rightCounting.fill(_)
self.assertEqual(leftCounting.entries, len(left))
self.assertEqual(rightCounting.entries, len(right))
finalResult = leftCounting + rightCounting
self.assertEqual(finalResult.entries, len(self.simple))
self.checkScaling(leftCounting)
self.checkScaling(leftCounting.toImmutable())
self.checkJson(leftCounting)
self.checkPickle(leftCounting)
self.checkName(leftCounting)
def setTH2sparse(sparse, yminBin, ymaxBin, th2):
for i, iindex in enumerate(xrange(sparse.minBin, sparse.maxBin + 1)):
for j, jindex in enumerate(xrange(yminBin, ymaxBin + 1)):
if iindex in sparse.bins and jindex in sparse.bins[iindex].bins:
th2.SetBinContent(i + 1, j + 1,
sparse.bins[iindex].bins[jindex].entries)
def _numpy(self, data, weights, shape):
q = self.quantity(data)
self._checkNPQuantity(q, shape)
self._checkNPWeights(weights, shape)
weights = self._makeNPWeights(weights, shape)
newentries = weights.sum()
import numpy
selection = numpy.isnan(q)
numpy.bitwise_not(selection, selection)
subweights = weights.copy()
subweights[selection] = 0.0
self.nanflow._numpy(data, subweights, shape)
# avoid nan warning in calculations by flinging the nans elsewhere
numpy.bitwise_not(selection, selection)
q = numpy.array(q, dtype=numpy.float64)
q[selection] = 0.0
weights = weights.copy()
weights[selection] = 0.0
if all(
isinstance(v, Count) and v.transform is identity
for c, v in self.bins) and numpy.all(
numpy.isfinite(q)) and numpy.all(numpy.isfinite(weights)):
h, _ = numpy.histogram(q, [float("-inf")] + [
(c1 + c2) / 2.0
for (c1, v1), (c2, v2) in zip(self.bins[:-1], self.bins[1:])
] + [float("inf")],
weights=weights)
for hi, (c, v) in zip(h, self.bins):
v.fill(None, float(hi))
else:
selection = numpy.empty(q.shape, dtype=numpy.bool)
selection2 = numpy.empty(q.shape, dtype=numpy.bool)
for index in xrange(len(self.bins)):
if index == 0:
high = (self.bins[index][0] +
self.bins[index + 1][0]) / 2.0
numpy.greater_equal(q, high, selection)
elif index == len(self.bins) - 1:
low = (self.bins[index - 1][0] + self.bins[index][0]) / 2.0
numpy.less(q, low, selection)
else:
low = (self.bins[index - 1][0] + self.bins[index][0]) / 2.0
high = (self.bins[index][0] +
self.bins[index + 1][0]) / 2.0
numpy.less(q, low, selection)
numpy.greater_equal(q, high, selection2)
numpy.bitwise_or(selection, selection2, selection)
subweights[:] = weights
subweights[selection] = 0.0
self.bins[index][1]._numpy(data, subweights, shape)
# no possibility of exception from here on out (for rollback)
self.entries += float(newentries)
def _cudaGenerateCode(self, parser, generator, inputFieldNames,
inputFieldTypes, derivedFieldTypes,
derivedFieldExprs, storageStructs, initCode,
initPrefix, initIndent, fillCode, fillPrefix,
fillIndent, combineCode, totalPrefix, itemPrefix,
combineIndent, jsonCode, jsonPrefix, jsonIndent,
weightVars, weightVarStack, tmpVarTypes,
suppressName):
normexpr = self._cudaQuantityExpr(parser, generator, inputFieldNames,
inputFieldTypes, derivedFieldTypes,
derivedFieldExprs, None)
initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) +
".entries = 0.0f;")
fillCode.append(" " * fillIndent + "atomicAdd(&" +
self._c99ExpandPrefix(*fillPrefix) + ".entries, " +
weightVarStack[-1] + ");")
combineCode.append(" " * combineIndent + "atomicAdd(&" +
self._c99ExpandPrefix(*totalPrefix) + ".entries, " +
self._c99ExpandPrefix(*itemPrefix) + ".entries);")
jsonCode.append(" " * jsonIndent +
"fprintf(out, \"{\\\"entries\\\": \");")
jsonCode.append(" " * jsonIndent + "floatToJson(out, " +
self._c99ExpandPrefix(*jsonPrefix) + ".entries);")
fillCode.append(" " * fillIndent +
"if (isnan({0})) {{".format(normexpr))
jsonCode.append(" " * jsonIndent +
"fprintf(out, \", \\\"nanflow:type\\\": \\\"" +
self.nanflow.name + "\\\"\");")
jsonCode.append(" " * jsonIndent +
"fprintf(out, \", \\\"nanflow\\\": \");")
self.nanflow._cudaGenerateCode(
parser, generator, inputFieldNames, inputFieldTypes,
derivedFieldTypes, derivedFieldExprs, storageStructs, initCode,
initPrefix + (("var", "nanflow"), ), initIndent + 2, fillCode,
fillPrefix + (("var", "nanflow"), ), fillIndent + 2, combineCode,
totalPrefix + (("var", "nanflow"), ),
itemPrefix + (("var", "nanflow"), ), combineIndent, jsonCode,
jsonPrefix + (("var", "nanflow"), ), jsonIndent, weightVars,
weightVarStack, tmpVarTypes, False)
fillCode.append(" " * fillIndent + "}")
fillCode.append(" " * fillIndent + "else {")
bin = "bin_" + str(len(tmpVarTypes))
tmpVarTypes[bin] = "int"
initCode.append(
" " * initIndent +
"for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.bins)))
fillCode.append(" " * fillIndent +
" const float edges[{0}] = {{{1}}};".format(
len(self.values) - 1, ", ".join(
floatToC99((self.bins[index - 1][0] +
self.bins[index][0]) / 2.0)
for index in xrange(1, len(self.bins)))))
fillCode.append(" " * fillIndent +
" for ({0} = 0; {0} < {1}; ++{0}) {{".format(
bin,
len(self.bins) - 1))
fillCode.append(" " * fillIndent +
" if ({0} < edges[{1}])".format(normexpr, bin))
fillCode.append(" " * fillIndent + " break;")
fillCode.append(" " * fillIndent + " }")
combineCode.append(
" " * combineIndent +
"for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.bins)))
jsonCode.append(" " * jsonIndent +
"fprintf(out, \", \\\"bins:type\\\": \\\"" +
self.bins[0][1].name + "\\\"\");")
if hasattr(self.bins[0][1],
"quantity") and self.bins[0][1].quantity.name is not None:
jsonCode.append(" " * jsonIndent +
"fprintf(out, \", \\\"bins:name\\\": \\\"" +
self.bins[0][1].quantity.name + "\\\"\");")
jsonCode.append(" " * jsonIndent + "{")
jsonCode.append(" " * jsonIndent +
" const float centers[{0}] = {{{1}}};".format(
len(self.values), ", ".join(
floatToC99(center)
for center, value in self.bins)))
jsonCode.append(" " * jsonIndent +
" fprintf(out, \", \\\"bins\\\": [\");")
jsonCode.append(" " * jsonIndent +
" for ({0} = 0; {0} < {1}; ++{0}) {{".format(
bin, len(self.values)))
jsonCode.append(" " * jsonIndent +
" fprintf(out, \"{\\\"center\\\": \");")
jsonCode.append(" " * jsonIndent + " floatToJson(out, centers[" +
bin + "]);")
jsonCode.append(" " * jsonIndent +
" fprintf(out, \", \\\"data\\\": \");")
self.bins[0][1]._cudaGenerateCode(
parser, generator, inputFieldNames, inputFieldTypes,
derivedFieldTypes, derivedFieldExprs, storageStructs, initCode,
initPrefix + (("var", "values"), ("index", bin)), initIndent + 2,
fillCode, fillPrefix + (("var", "values"), ("index", bin)),
fillIndent + 2, combineCode, totalPrefix + (("var", "values"),
("index", bin)),
itemPrefix + (("var", "values"),
("index", bin)), combineIndent + 2, jsonCode,
jsonPrefix + (("var", "values"), ("index", bin)), jsonIndent + 4,
weightVars, weightVarStack, tmpVarTypes, True)
initCode.append(" " * initIndent + "}")
fillCode.append(" " * fillIndent + "}")
combineCode.append(" " * combineIndent + "}")
jsonCode.append(" " * jsonIndent + " fprintf(out, \"}\");")
jsonCode.append(" " * jsonIndent +
" if ({0} != {1})".format(bin,
len(self.values) - 1))
jsonCode.append(" " * jsonIndent + " fprintf(out, \", \");")
jsonCode.append(" " * jsonIndent + " }")
jsonCode.append(" " * jsonIndent + "}")
if suppressName or self.quantity.name is None:
jsonCode.append(" " * jsonIndent + "fprintf(out, \"]}\");")
else:
jsonCode.append(" " * jsonIndent +
"fprintf(out, \"], \\\"name\\\": " +
json.dumps(json.dumps(self.quantity.name))[1:-1] +
"}\");")
storageStructs[self._c99StructName()] = """
typedef struct {{
float entries;
{3} nanflow;
{1} values[{2}];
}} {0};
""".format(self._c99StructName(), self.bins[0][1]._cudaStorageType(),
len(self.values), self.nanflow._cudaStorageType())
def _clingUpdate(self, filler, *extractorPrefix):
obj = self._clingExpandPrefix(filler, *extractorPrefix)
self.entries += obj.entries
for i in xrange(len(self.values)):
self.bins[i][1]._clingUpdate(obj, ("func", ["getValues", i]))
self.nanflow._clingUpdate(obj, ("var", "nanflow"))
def _c99GenerateCode(self, parser, generator, inputFieldNames,
inputFieldTypes, derivedFieldTypes, derivedFieldExprs,
storageStructs, initCode, initPrefix, initIndent,
fillCode, fillPrefix, fillIndent, weightVars,
weightVarStack, tmpVarTypes):
normexpr = self._c99QuantityExpr(parser, generator, inputFieldNames,
inputFieldTypes, derivedFieldTypes,
derivedFieldExprs, None)
initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) +
".entries = 0.0;")
fillCode.append(" " * fillIndent + self._c99ExpandPrefix(*fillPrefix) +
".entries += " + weightVarStack[-1] + ";")
fillCode.append(" " * fillIndent +
"if (std::isnan({0})) {{".format(normexpr))
self.nanflow._c99GenerateCode(
parser, generator, inputFieldNames, inputFieldTypes,
derivedFieldTypes, derivedFieldExprs, storageStructs, initCode,
initPrefix + (("var", "nanflow"), ), initIndent, fillCode,
fillPrefix + (("var", "nanflow"), ), fillIndent + 2, weightVars,
weightVarStack, tmpVarTypes)
fillCode.append(" " * fillIndent + "}")
fillCode.append(" " * fillIndent + "else {")
bin = "bin_" + str(len(tmpVarTypes))
tmpVarTypes[bin] = "int"
initCode.append(
" " * initIndent +
"for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.bins)))
fillCode.append(" " * fillIndent +
" const double edges[{0}] = {{{1}}};".format(
len(self.values) - 1, ", ".join(
floatToC99((self.bins[idx - 1][0] +
self.bins[idx][0]) / 2.0)
for idx in xrange(1, len(self.bins)))))
fillCode.append(" " * fillIndent +
" for ({0} = 0; {0} < {1}; ++{0}) {{".format(
bin,
len(self.bins) - 1))
fillCode.append(" " * fillIndent +
" if ({0} < edges[{1}])".format(normexpr, bin))
fillCode.append(" " * fillIndent + " break;")
fillCode.append(" " * fillIndent + " }")
self.bins[0][1]._c99GenerateCode(
parser, generator, inputFieldNames, inputFieldTypes,
derivedFieldTypes, derivedFieldExprs, storageStructs, initCode,
initPrefix + (("var", "values"), ("index", bin)), initIndent + 2,
fillCode, fillPrefix + (("var", "values"), ("index", bin)),
fillIndent + 2, weightVars, weightVarStack, tmpVarTypes)
initCode.append(" " * initIndent + "}")
fillCode.append(" " * fillIndent + "}")
storageStructs[self._c99StructName()] = """
typedef struct {{
double entries;
{3} nanflow;
{1} values[{2}];
{1}& getValues(int i) {{ return values[i]; }}
}} {0};
""".format(self._c99StructName(), self.bins[0][1]._c99StorageType(),
len(self.values), self.nanflow._c99StorageType())
def _cudaGenerateCode(self, parser, generator, inputFieldNames,
inputFieldTypes, derivedFieldTypes,
derivedFieldExprs, storageStructs, initCode,
initPrefix, initIndent, fillCode, fillPrefix,
fillIndent, combineCode, totalPrefix, itemPrefix,
combineIndent, jsonCode, jsonPrefix, jsonIndent,
weightVars, weightVarStack, tmpVarTypes,
suppressName):
tmpJsonCode = []
if isinstance(self, (Label, Index)):
tmpJsonCode.append(" " * jsonIndent +
"fprintf(out, \"{\\\"sub:type\\\": \\\"" +
self.values[0].name +
"\\\", \\\"data\\\": \");")
else:
tmpJsonCode.append(" " * jsonIndent +
"fprintf(out, \"{\\\"data\\\": \");")
if isinstance(self, (Label, UntypedLabel)):
tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"{\");")
else:
tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"[\");")
if isinstance(self, Branch):
rightOrder = {}
lastJsonCode = len(tmpJsonCode)
last = None
n = 0
i = 0
for s, k, v in self._c99CanonicalOrder(self.pairs.items()):
if not isinstance(self, Branch) and last is not None:
tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \", \");")
if isinstance(self, (Label, UntypedLabel)):
tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"" +
json.dumps(json.dumps(k))[1:-1] + ": \");")
if isinstance(self, (UntypedLabel, Branch)):
tmpJsonCode.append(" " * jsonIndent +
"fprintf(out, \"{\\\"type\\\": \\\"" +
v.name + "\\\", \\\"data\\\": \");")
if last is not None and s != last:
n += 1
i = 0
v._cudaGenerateCode(
parser, generator, inputFieldNames, inputFieldTypes,
derivedFieldTypes, derivedFieldExprs, storageStructs, initCode,
initPrefix + (("var", "sub" + str(n)), ("index", i)),
initIndent, fillCode, fillPrefix + (("var", "sub" + str(n)),
("index", i)), fillIndent,
combineCode, totalPrefix + (("var", "sub" + str(n)),
("index", i)),
itemPrefix + (("var", "sub" + str(n)),
("index", i)), combineIndent, tmpJsonCode,
jsonPrefix + (("var", "sub" + str(n)),
("index", i)), jsonIndent, weightVars,
weightVarStack, tmpVarTypes, suppressName)
i += 1
last = s
if isinstance(self, (UntypedLabel, Branch)):
tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"}\");")
if isinstance(self, Branch):
rightOrder[k] = tmpJsonCode[lastJsonCode:]
tmpJsonCode = tmpJsonCode[:lastJsonCode]
if isinstance(self, Branch):
for k in xrange(len(rightOrder)):
if k != 0:
tmpJsonCode.append(" " * jsonIndent +
"fprintf(out, \", \");")
tmpJsonCode.extend(rightOrder[k])
if isinstance(self, (Label, UntypedLabel)):
tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"}\");")
else:
tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"]\");")
initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) +
".entries = 0.0f;")
fillCode.append(" " * fillIndent + "atomicAdd(&" +
self._c99ExpandPrefix(*fillPrefix) + ".entries, " +
weightVarStack[-1] + ");")
combineCode.append(" " * combineIndent + "atomicAdd(&" +
self._c99ExpandPrefix(*totalPrefix) + ".entries, " +
self._c99ExpandPrefix(*itemPrefix) + ".entries);")
tmpJsonCode.append(" " * jsonIndent +
"fprintf(out, \", \\\"entries\\\": \");")
tmpJsonCode.append(" " * jsonIndent + "floatToJson(out, " +
self._c99ExpandPrefix(*jsonPrefix) + ".entries);")
tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"}\");")
jsonCode.extend(tmpJsonCode)
storageStructs[self._c99StructName()] = self._cudaStruct()
def _cppGenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes,
derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix,
fillIndent, weightVars, weightVarStack, tmpVarTypes):
normexpr = self._c99QuantityExpr(
parser,
generator,
inputFieldNames,
inputFieldTypes,
derivedFieldTypes,
derivedFieldExprs,
None)
initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".entries = 0.0;")
initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".values.clear();")
fillCode.append(" " * fillIndent + self._c99ExpandPrefix(*fillPrefix) +
".entries += " + weightVarStack[-1] + ";")
fillCode.append("""{indent}if ({values}.find({q}) == {values}.end())
{indent} {values}[{q}] = 0.0;
{indent}{values}[{q}] += {weight};""".format(
indent=" " * fillIndent,
values=self._c99ExpandPrefix(*fillPrefix) + ".values",
q=normexpr,
weight=weightVarStack[-1]
))
if self.range[0] == "N" and len(self.range) > 1:
storageStructs[self.range] = """
class {0} {{
public:
double {1};
{0}({2}): {3} {{ }}
{0}(const {0}& other): {4} {{ }}
{0}(): {5} {{ }}
Bool_t operator<(const {0}& other) const {{
{6}
else return false;
}}
}};
""".format(self.range,
", ".join("v" + str(i) for i in xrange(self.dimension)),
", ".join("double v" + str(i) for i in xrange(self.dimension)),
", ".join("v" + str(i) + "(v" + str(i) + ")" for i in xrange(self.dimension)),
", ".join("v" + str(i) + "(other.v" + str(i) + ")" for i in xrange(self.dimension)),
", ".join("v" + str(i) + "(0.0)" for i in xrange(self.dimension)),
"\n ".join(
("else " if i != 0 else "") +
"if (v" +
str(i) +
" < other.v" +
str(i) +
") return true;" for i in xrange(
self.dimension))
)
storageStructs[self._c99StructName()] = """
typedef struct {{
double entries;
std::map<{1}, double> values;
double getValues({1} i) {{ return values[i]; }}
}} {0};
""".format(self._c99StructName(), "double" if self.range == "N" else "std::string" if self.range == "S" else self.range)
