def testWelchsTTest_EmptySample_RaisesError(self):
"""An error should be raised when an empty sample is passed in."""
with self.assertRaises(RuntimeError):
ttest.WelchsTTest([], [])
with self.assertRaises(RuntimeError):
ttest.WelchsTTest([], [1, 2, 3])
with self.assertRaises(RuntimeError):
ttest.WelchsTTest([1, 2, 3], [])
def MakeChangePoint(series, split_index):
"""Makes a ChangePoint object for the given series at the given point.
Args:
series: A list of (x, y) pairs.
split_index: Index of the first point after the split.
Returns:
A ChangePoint object.
"""
assert 0 <= split_index < len(series)
x_values, y_values = zip(*series)
left, right = y_values[:split_index], y_values[split_index:]
left_median, right_median = math_utils.Median(left), math_utils.Median(
right)
ttest_results = ttest.WelchsTTest(left, right)
return ChangePoint(
x_value=x_values[split_index],
median_before=left_median,
median_after=right_median,
size_before=len(left),
size_after=len(right),
window_start=x_values[0],
window_end=x_values[-1], # inclusive bound
relative_change=math_utils.RelativeChange(left_median, right_median),
std_dev_before=math_utils.StandardDeviation(left),
t_statistic=ttest_results.t,
degrees_of_freedom=ttest_results.df,
p_value=ttest_results.p)
def testWelchsTTest(self):
"""Tests the t value and degrees of freedom output of Welch's t-test."""
# The t-value can be checked with scipy.stats.ttest_ind(equal_var=False).
# However the t-value output by scipy.stats.ttest_ind is -6.32455532034.
# This implementation produces slightly different results.
result = ttest.WelchsTTest([2, 3, 2, 3, 2, 3], [4, 5, 4, 5, 4, 5])
self.assertAlmostEqual(10.0, result.df)
self.assertAlmostEqual(-6.325, result.t, delta=1.0)
def testTTest_DifferentMeanDifference(self):
"""Verifies that smaller difference between means -> higher p value."""
result_far_means = ttest.WelchsTTest([2, 3, 2, 3], [5, 6, 5, 6])
result_near_means = ttest.WelchsTTest([2, 3, 2, 3], [3, 4, 3, 4])
self.assertLess(result_far_means.p, result_near_means.p)
def testTTest_DifferentSampleSize(self):
"""Verifies that smaller sample size -> higher p value."""
result_larger_sample = ttest.WelchsTTest([2, 3, 2, 3], [4, 5, 4, 5])
result_smaller_sample = ttest.WelchsTTest([2, 3, 2, 3], [4, 5])
self.assertLess(result_larger_sample.p, result_smaller_sample.p)
def testTTest_DifferentVariance(self):
"""Verifies that higher variance -> higher p value."""
result_low_var = ttest.WelchsTTest([2, 3, 2, 3], [4, 5, 4, 5])
result_high_var = ttest.WelchsTTest([1, 4, 1, 4], [3, 6, 3, 6])
self.assertLess(result_low_var.p, result_high_var.p)
def testTTest_VeryDifferentSamples_PValueIsLow(self):
"""Checks that p is very low when the samples are clearly different."""
result = ttest.WelchsTTest([100, 101, 100, 101, 100],
[1, 2, 1, 2, 1, 2, 1, 2])
self.assertLessEqual(250, result.t)
self.assertLessEqual(0.01, result.p)
def testTTest_EqualSamples_PValueIsOne(self):
"""Checks that t = 0 and p = 1 when the samples are the same."""
result = ttest.WelchsTTest([1, 2, 3], [1, 2, 3])
self.assertEqual(0, result.t)
self.assertEqual(1, result.p)
