def test_accumulators():
a = processor.value_accumulator(float)
a += 3.0
assert a.value == 3.0
assert a.identity().value == 0.0
a = processor.value_accumulator(partial(np.array, [2.0]))
a += 3.0
assert np.array_equal(a.value, np.array([5.0]))
assert np.array_equal(a.identity().value, np.array([2.0]))
lacc = processor.list_accumulator(range(4))
lacc += [3]
lacc += processor.list_accumulator([1, 2])
assert lacc == [0, 1, 2, 3, 3, 1, 2]
b = processor.set_accumulator({"apples", "oranges"})
b += {"pears"}
b += "grapes"
assert b == {"apples", "oranges", "pears", "grapes"}
c = processor.dict_accumulator({"num": a, "fruit": b})
c["num"] += 2.0
c += processor.dict_accumulator({
"num2":
processor.value_accumulator(int),
"fruit":
processor.set_accumulator({"apples", "cherries"}),
})
assert c["num2"].value == 0
assert np.array_equal(c["num"].value, np.array([7.0]))
assert c["fruit"] == {"apples", "oranges", "pears", "grapes", "cherries"}
d = processor.defaultdict_accumulator(float)
d["x"] = 0.0
d["x"] += 4.0
d["y"] += 5.0
d["z"] += d["x"]
d["x"] += d["y"]
assert d["x"] == 9.0
assert d["y"] == 5.0
assert d["z"] == 4.0
assert d["w"] == 0.0
f = processor.defaultdict_accumulator(lambda: 2.0)
f["x"] += 4.0
assert f["x"] == 6.0
f += f
assert f["x"] == 12.0
assert f["y"] == 2.0
a = processor.column_accumulator(np.arange(6).reshape(2, 3))
b = processor.column_accumulator(np.arange(12).reshape(4, 3))
a += b
assert a.value.sum() == 81
def test_accumulators():
a = processor.value_accumulator(float)
a += 3.
assert a.value == 3.
assert a.identity().value == 0.
a = processor.value_accumulator(partial(np.array, [2.]))
a += 3.
assert np.array_equal(a.value, np.array([5.]))
assert np.array_equal(a.identity().value, np.array([2.]))
l = processor.list_accumulator(range(4))
l += [3]
l += processor.list_accumulator([1, 2])
assert l == [0, 1, 2, 3, 3, 1, 2]
b = processor.set_accumulator({'apples', 'oranges'})
b += {'pears'}
b += 'grapes'
assert b == {'apples', 'oranges', 'pears', 'grapes'}
c = processor.dict_accumulator({'num': a, 'fruit': b})
c['num'] += 2.
c += processor.dict_accumulator({
'num2': processor.value_accumulator(int),
'fruit': processor.set_accumulator({'apples', 'cherries'}),
})
assert c['num2'].value == 0
assert np.array_equal(c['num'].value, np.array([7.]))
assert c['fruit'] == {'apples', 'oranges', 'pears', 'grapes', 'cherries'}
d = processor.defaultdict_accumulator(float)
d['x'] = 0.
d['x'] += 4.
d['y'] += 5.
d['z'] += d['x']
d['x'] += d['y']
assert d['x'] == 9.
assert d['y'] == 5.
assert d['z'] == 4.
assert d['w'] == 0.
e = d + c
f = processor.defaultdict_accumulator(lambda: 2.)
f['x'] += 4.
assert f['x'] == 6.
f += f
assert f['x'] == 12.
assert f['y'] == 2.
a = processor.column_accumulator(np.arange(6).reshape(2,3))
b = processor.column_accumulator(np.arange(12).reshape(4,3))
a += b
assert a.value.sum() == 81
def test_accumulators():
a = processor.accumulator(0.)
a += 3.
a += processor.accumulator(2)
assert a.value == 5.
assert a.identity().value == 0.
a = processor.accumulator(np.array([0.]))
a += 3.
a += processor.accumulator(2)
assert a.value == np.array([5.])
assert a.identity().value == np.array([0.])
b = processor.set_accumulator({'apples', 'oranges'})
b += {'pears'}
b += 'grapes'
assert b == {'apples', 'oranges', 'pears', 'grapes'}
c = processor.dict_accumulator({'num': a, 'fruit': b})
c['num'] += 2.
c += processor.dict_accumulator({
'num2':
processor.accumulator(0),
'fruit':
processor.set_accumulator({'apples', 'cherries'}),
})
assert c['num2'].value == 0
assert c['num'].value == 7.
assert c['fruit'] == {'apples', 'oranges', 'pears', 'grapes', 'cherries'}
d = processor.defaultdict_accumulator(lambda: processor.accumulator(0.))
d['x'] = processor.accumulator(0.)
d['x'] += 4.
d['y'] += 5.
d['z'] += d['x']
d['x'] += d['y']
assert d['x'].value == 9.
assert d['y'].value == 5.
assert d['z'].value == 4.
assert d['w'].value == 0.
e = d + c
f = processor.defaultdict_accumulator(lambda: 2.)
f['x'] += 4.
assert f['x'] == 6.
f += f
assert f['x'] == 12.
assert f['y'] == 2.
def __init__(self, columns=[], canaries=[]):
self._columns = columns
self._canaries = canaries
dataset_axis = hist.Cat("dataset", "Primary dataset")
mass_axis = hist.Bin("mass", r"$m_{\mu\mu}$ [GeV]", 30000, 0.25, 300)
pt_axis = hist.Bin("pt", r"$p_{T}$ [GeV]", 30000, 0.25, 300)
self._accumulator = processor.dict_accumulator(
{
"mass": hist.Hist("Counts", dataset_axis, mass_axis),
"pt": hist.Hist("Counts", dataset_axis, pt_axis),
"cutflow": processor.defaultdict_accumulator(int),
"worker": processor.set_accumulator(),
}
)
def get_badfiles(item):
import coffea.processor as processor
out = processor.set_accumulator()
# add timeout option according to modified uproot numentries defaults
xrootdsource = {
"timeout": None,
"chunkbytes": 32 * 1024,
"limitbytes": 1024**2,
"parallel": False
}
try:
# Test ability to open file and read tree
file = uproot.open(item.filename, xrootdsource=xrootdsource)
tree = file[item.treename]
entries = tree.numentries
except Exception as e:
out.add((item.dataset, item.filename))
return out
print("File open error for %s, %s" % (ds, fn))
exit(0)
processor_instance = load(args.processor)
combined_accumulator = processor.dict_accumulator({
'stats':
processor.dict_accumulator({
'nentries':
processor.value_accumulator(int),
'bytesread':
processor.value_accumulator(int),
'sumworktime':
processor.value_accumulator(float),
'columns_accessed':
processor.set_accumulator(),
}),
'job':
processor_instance.accumulator.identity(),
})
def work_function(item):
dataset, file = item
out, stats = process_file(dataset, file, processor_instance,
combined_accumulator['stats'], preload_items,
args.stride)
return processor.dict_accumulator({'stats': stats, 'job': out})
tstart = time.time()
if args.executor == 'iterative':
if args.profilehtml is not None: