def test_index_means2():
# Suppose we have three levels, so n = 3.
# For the sake of example, level 1 is repetitions, level 2 is executions,
# and level 3 is compilations. Now suppose we repeat level 3 twice,
# level 2 twice and level 3 five times.
#
# This would be a valid data set:
# Note that the indicies are off-by-one due to python indicies starting
# from 0.
d = Data(
{
(0, 0): [3, 4, 4, 1, 2], # times for compile 1, execution 1
(0, 1): [3, 3, 3, 3, 3], # compile 1, execution 2
(1, 0): [1, 2, 3, 4, 5], # compile 2, execution 1
(1, 1): [1, 1, 4, 4, 1], # compile 2, execution 2
},
[2, 2, 5]) # counts for each level (highest to lowest)
# By calling mean with an empty tuple we compute the mean at all levels
# i.e. the mean of all times:
x = [3, 4, 4, 1, 2, 3, 3, 3, 3, 3, 1, 2, 3, 4, 5, 1, 1, 4, 4, 1]
expect = sum(x) / float(len(x))
assert expect == d.mean(())
# By calling with a singleton tuple we compute the mean for a given
#compilation. E.g. compilation 2
x = [1, 2, 3, 4, 5, 1, 1, 4, 4, 1]
expect = sum(x) / float(len(x))
assert expect == d.mean((1, ))
# By calling with a pair we compute the mean for a given compile
# and execution combo.
# E.g. compile 1, execution 2, which is obviously a mean of 3.
assert d.mean((0, 1)) == 3
def test_index_iter():
d = Data({(0, 0): [1, 2, 3, 4, 5], (0, 1): [3, 4, 5, 6, 7]}, [1, 2, 5])
assert list(d.index_iterator()) == [
(0, 0, 0),
(0, 0, 1),
(0, 0, 2),
(0, 0, 3),
(0, 0, 4),
(0, 1, 0),
(0, 1, 1),
(0, 1, 2),
(0, 1, 3),
(0, 1, 4),
]
assert list(d.index_iterator(start=1)) == [
(0, 0),
(0, 1),
(0, 2),
(0, 3),
(0, 4),
(1, 0),
(1, 1),
(1, 2),
(1, 3),
(1, 4),
]
assert list(d.index_iterator(start=0, stop=1)) == [(0, )]
assert list(d.index_iterator(start=1, stop=2)) == [(0, ), (1, )]
def test_index_means():
d = Data({(0, 0): [0, 2]}, [1, 1, 2])
assert d.mean(()) == 1
assert d.mean((0, 0)) == 1
assert d.mean((0, 0, 0)) == d[0, 0, 0]
assert d.mean((0, 0, 1)) == d[0, 0, 1]
def test_worked_example_3_level():
# three level experiment
# This is the worked example from the paper.
data = Data(
{
(0, 0): [9., 5.],
(0, 1): [8., 3.],
(1, 0): [10., 6.],
(1, 1): [7., 11.],
(2, 0): [1., 12.],
(2, 1): [2., 4.],
}, [3, 2, 2])
correct = {
(0, 0): 7.0,
(0, 1): 5.5,
(1, 0): 8.0,
(1, 1): 9.0,
(2, 0): 6.5,
(2, 1): 3.0,
}
for index in data.index_iterator(stop=2):
assert data.mean(index) == correct[index]
assert data.mean() == 6.5
assert round(data.Si2(1), 1) == 16.5
assert round(data.Si2(2), 1) == 2.6
assert round(data.Si2(3), 1) == 3.6
assert round(data.Ti2(1), 1) == 16.5
assert round(data.Ti2(2), 1) == -5.7
assert round(data.Ti2(3), 1) == 2.3
def test_bootstrap():
# XXX needs info on how expected val was computed
data = Data(
{
(0, ): [2.5, 3.1, 2.7],
(1, ): [5.1, 1.1, 2.3],
(2, ): [4.7, 5.5, 7.1],
}, [3, 3])
random.seed(1)
expect = 4.8111111111
got = data.bootstrap_means(1) # one iteration
assert abs(got[0] - expect) <= 0.0001
def test_si2_bigger_example():
# Let's compute S_1^2 for the following data
d = Data(
{
(0, 0): [3, 4, 3],
(0, 1): [1.2, 3.1, 3],
(1, 0): [0.2, 1, 1.5],
(1, 1): [1, 2, 3]
}, [2, 2, 3])
# So we have n = 3, r = (2, 2, 3)
# By my reckoning we should get something close to 0.72667 (working below)
# XXX Explanation from whiteboard need to go here XXX
assert abs(d.Si2(1) - 0.72667) <= 0.0001
def test_confidence_quotient_div_zero():
data1 = Data(
{
(0, ): [2.5, 3.1, 2.7],
(1, ): [5.1, 1.1, 2.3],
(2, ): [4.7, 5.5, 7.1],
}, [3, 3])
data2 = Data({ # This has a mean of zero
(0, ) : [ 0, 0, 0],
(1, ) : [ 0, 0, 0],
(2, ) : [ 0, 0, 0],
}, [3, 3])
# Since all ratios will be +inf, the median should also be +inf
(_, median, _) = data1.bootstrap_quotient(data2, iterations=1)
assert median == float("inf")
def test_optimal_reps_with_rounding():
""" Same as test_optimal_reps_no_rounding() just with rounding."""
d = Data(
{
(0, 0): [3, 4, 3],
(0, 1): [1.2, 3.1, 3],
(1, 0): [0.2, 1, 1.5],
(1, 1): [1, 2, 3]
}, [2, 2, 3])
got = [d.optimalreps(i, (100, 20, 3)) for i in [1, 2]]
expect = [5, 2]
for i in range(len(got)):
assert got[i] == expect[i]
for i in got:
assert type(i) == int
def test_ti2():
# To verify this, consider the following data:
d = Data(
{
(0, 0): [3, 4, 3],
(0, 1): [1.2, 3.1, 3],
(1, 0): [0.2, 1, 1.5],
(1, 1): [1, 2, 3]
}, [2, 2, 3])
# Let's manually look at S_i^2 where 1 <= i <= n:
#si_vec = [ d.Si2(i) for i in range(1, 4) ]
#print(si_vec)
ti_vec = [d.Ti2(i) for i in range(1, 4)]
expect = [0.7266667, 0.262777778, 0.7747]
for i in range(len(expect)):
assert abs(ti_vec[i] - expect[i]) <= 0.0001
def test_geomean_data():
data1 = Data(
{
(0, ): [2.9, 3.1, 3.0],
(1, ): [3.1, 2.6, 3.3],
(2, ): [3.2, 3.0, 2.9],
}, [3, 3])
data2 = Data(
{
(0, ): [3.9, 4.1, 4.0],
(1, ): [4.1, 3.6, 4.3],
(2, ): [4.2, 4.0, 3.9],
}, [3, 3])
(_, mean1, _) = data1.bootstrap_quotient(data2)
(_, mean2, _) = bootstrap_geomean([data1], [data2])
assert round(mean1, 3) == round(mean2, 3)
(_, mean, _) = bootstrap_geomean([data1, data2], [data2, data1])
assert round(mean, 5) == 1.0
def test_optimal_reps_no_rounding():
d = Data(
{
(0, 0): [3, 4, 3],
(0, 1): [1.2, 3.1, 3],
(1, 0): [0.2, 1, 1.5],
(1, 1): [1, 2, 3]
}, [2, 2, 3])
#ti_vec = [ d.Ti2(i) for i in range (1, 4) ]
#print(ti_vec)
# And suppose the costs (high level to low) are 100, 20 and 3 (seconds)
# By my reckoning, the optimal repetition counts should be r_1 = 5, r_2 = 2
# XXX show working XXX
got = [d.optimalreps(i, (100, 20, 3), round=False) for i in [1, 2]]
expect = [4.2937, 1.3023]
for i in range(len(got)):
assert abs(got[i] - expect[i]) <= 0.001
for i in got:
assert type(i) == float
def test_confidence_quotient():
data1 = Data(
{
(0, ): [2.5, 3.1, 2.7],
(1, ): [5.1, 1.1, 2.3],
(2, ): [4.7, 5.5, 7.1],
}, [3, 3])
data2 = Data(
{
(0, ): [3.5, 4.1, 3.7],
(1, ): [6.1, 2.1, 3.3],
(2, ): [5.7, 6.5, 8.1],
}, [3, 3])
random.seed(1)
a = data1._bootstrap_sample()
b = data2._bootstrap_sample()
random.seed(1)
(_, mean, _) = data1.bootstrap_quotient(data2, iterations=1)
assert mean == _mean(a) / _mean(b)
def test_worked_example_2_level():
data = Data(
{
(0, ): [9., 5., 8., 3.],
(1, ): [10., 6., 7., 11.],
(2, ): [1., 12., 2., 4.],
}, [3, 4])
correct = {
(0, ): 6.3,
(1, ): 8.5,
(2, ): 4.8,
}
for index in data.index_iterator(stop=1):
assert round(data.mean(index), 1) == correct[index]
assert data.mean() == 6.5
assert round(data.Si2(1), 1) == 12.7
assert round(data.Si2(2), 1) == 3.6
assert round(data.Ti2(1), 1) == 12.7
assert round(data.Ti2(2), 1) == 0.4
def test_rep_levels():
d = Data({(0, 0): [1, 2, 3, 4, 5], (0, 1): [3, 4, 5, 6, 7]}, [1, 2, 5])
assert d.r(1) == 5 # lowest level, i.e. arity of the lists in the map
assert d.r(2) == 2
assert d.r(3) == 1
# indexs are one based, so 0 or less is invalid
with pytest.raises(AssertionError):
d.r(0)
with pytest.raises(AssertionError):
d.r(-1337)
# Since we have 3 levels here, levels 4 and above are bogus
with pytest.raises(AssertionError):
d.r(4)
with pytest.raises(AssertionError):
d.r(666)
def test_indicies():
d = Data({(0, 0): [1, 2, 3, 4, 5], (0, 1): [3, 4, 5, 6, 7]}, [1, 2, 5])
assert d[0, 0, 0] == 1
assert d[0, 0, 4] == 5
assert d[0, 1, 2] == 5
def test_si2():
d = Data({(0, 0): [0, 0]}, [1, 1, 2])
assert d.Si2(1) == 0