def test_calculate_results_for_three_test_variants_almost_equal(self):
variant_control = Variant("B", 130, 65)
variant_test_1 = Variant("A", 120, 60)
variant_test_2 = Variant("A", 110, 52)
variant_test_3 = Variant("A", 100, 46)
probabilities = ClickhouseFunnelExperimentResult.calculate_results(
variant_control, [variant_test_1, variant_test_2, variant_test_3])
self.assertAlmostEqual(sum(probabilities), 1)
self.assertAlmostEqual(probabilities[0], 0.168, places=1)
self.assertAlmostEqual(probabilities[1], 0.174, places=1)
self.assertAlmostEqual(probabilities[2], 0.292, places=1)
self.assertAlmostEqual(probabilities[3], 0.365, places=1)
alternative_probability_for_control = calculate_probability_of_winning_for_target(
variant_control, [variant_test_1, variant_test_2, variant_test_3])
self.assertAlmostEqual(probabilities[0],
alternative_probability_for_control,
places=1)
self.assertAlmostEqual(calculate_expected_loss(
variant_test_2, [variant_control, variant_test_1, variant_test_3]),
0.033,
places=2)
# passing in artificial probabilities to subvert the low_probability threshold
significant, loss = ClickhouseFunnelExperimentResult.are_results_significant(
variant_control, [variant_test_1, variant_test_2, variant_test_3],
[1, 0])
self.assertAlmostEqual(loss, 0.012, places=2)
self.assertEqual(significant, ExperimentSignificanceCode.HIGH_LOSS)
def test_calculate_results_for_three_test_variants(self):
variant_test_1 = Variant("A", 100, 10)
variant_test_2 = Variant("A", 100, 3)
variant_test_3 = Variant("A", 100, 30)
variant_control = Variant("B", 100, 18)
probabilities = ClickhouseFunnelExperimentResult.calculate_results(
variant_control, [variant_test_1, variant_test_2, variant_test_3])
self.assertAlmostEqual(sum(probabilities), 1)
self.assertAlmostEqual(probabilities[0], 0.0, places=1)
self.assertAlmostEqual(probabilities[1], 0.033, places=1)
self.assertAlmostEqual(probabilities[2], 0.967, places=1)
self.assertAlmostEqual(probabilities[3], 0.0, places=1)
alternative_probability_for_control = calculate_probability_of_winning_for_target(
variant_control, [variant_test_1, variant_test_2, variant_test_3])
self.assertAlmostEqual(probabilities[0],
alternative_probability_for_control,
places=1)
self.assertAlmostEqual(calculate_expected_loss(
variant_test_2, [variant_control, variant_test_1, variant_test_3]),
0.0004,
places=2)
significant, loss = ClickhouseFunnelExperimentResult.are_results_significant(
variant_control, [variant_test_1, variant_test_2, variant_test_3],
probabilities)
self.assertAlmostEqual(loss, 0.0004, places=2)
self.assertEqual(significant, ExperimentSignificanceCode.SIGNIFICANT)
def test_calculate_results_for_two_test_variants_almost_equal(self):
variant_test_1 = Variant("A", 120, 60)
variant_test_2 = Variant("A", 110, 52)
variant_control = Variant("B", 130, 65)
probabilities = ClickhouseFunnelExperimentResult.calculate_results(
variant_control, [variant_test_1, variant_test_2])
self.assertAlmostEqual(sum(probabilities), 1)
self.assertAlmostEqual(probabilities[0], 0.277, places=1)
self.assertAlmostEqual(probabilities[1], 0.282, places=1)
self.assertAlmostEqual(probabilities[2], 0.440, places=1)
alternative_probability_for_control = calculate_probability_of_winning_for_target(
variant_control, [variant_test_1, variant_test_2])
self.assertAlmostEqual(probabilities[0],
alternative_probability_for_control,
places=1)
self.assertAlmostEqual(calculate_expected_loss(
variant_test_2, [variant_control, variant_test_1]),
0.022,
places=2)
significant, loss = ClickhouseFunnelExperimentResult.are_results_significant(
variant_control, [variant_test_1, variant_test_2], probabilities)
self.assertAlmostEqual(loss, 1, places=3)
self.assertEqual(significant,
ExperimentSignificanceCode.LOW_WIN_PROBABILITY)
def test_calculate_results(self):
variant_test = Variant("A", 100, 10)
variant_control = Variant("B", 100, 18)
_, probability = ClickhouseFunnelExperimentResult.calculate_results(
variant_control, [variant_test])
self.assertAlmostEqual(probability, 0.918, places=2)
significant, loss = ClickhouseFunnelExperimentResult.are_results_significant(
variant_control, [variant_test], [probability])
self.assertAlmostEqual(loss, 0.0016, places=3)
self.assertEqual(significant, ExperimentSignificanceCode.SIGNIFICANT)
def test_absolute_loss_less_than_one_percent_but_not_significant(self):
variant_test_1 = Variant("A", 286, 2014)
variant_control = Variant("B", 267, 2031)
probabilities = ClickhouseFunnelExperimentResult.calculate_results(
variant_control, [variant_test_1])
self.assertAlmostEqual(sum(probabilities), 1)
self.assertAlmostEqual(probabilities[0], 0.197, places=1)
self.assertAlmostEqual(probabilities[1], 0.802, places=1)
self.assertAlmostEqual(calculate_expected_loss(variant_test_1,
[variant_control]),
0.0010,
places=3)
significant, loss = ClickhouseFunnelExperimentResult.are_results_significant(
variant_control, [variant_test_1], probabilities)
self.assertAlmostEqual(loss, 1, places=3)
self.assertEqual(significant,
ExperimentSignificanceCode.LOW_WIN_PROBABILITY)
def test_simulation_result_is_close_to_closed_form_solution(self):
variant_test = Variant("A", 100, 10)
variant_control = Variant("B", 100, 18)
_, probability = ClickhouseFunnelExperimentResult.calculate_results(
variant_control, [variant_test])
self.assertAlmostEqual(probability, 0.918, places=1)
alternative_probability = calculate_probability_of_winning_for_target(
variant_test, [variant_control])
self.assertAlmostEqual(probability, alternative_probability, places=1)
def test_calculate_results_for_three_test_variants_much_better_than_control(
self):
variant_control = Variant("B", 80, 65)
variant_test_1 = Variant("A", 130, 60)
variant_test_2 = Variant("A", 135, 62)
variant_test_3 = Variant("A", 132, 60)
probabilities = ClickhouseFunnelExperimentResult.calculate_results(
variant_control, [variant_test_1, variant_test_2, variant_test_3])
self.assertAlmostEqual(sum(probabilities), 1)
alternative_probability_for_control = calculate_probability_of_winning_for_target(
variant_control, [variant_test_1, variant_test_2, variant_test_3])
self.assertAlmostEqual(probabilities[0],
alternative_probability_for_control,
places=1)
significant, loss = ClickhouseFunnelExperimentResult.are_results_significant(
variant_control, [variant_test_1, variant_test_2, variant_test_3],
probabilities)
self.assertAlmostEqual(loss, 0, places=2)
self.assertEqual(significant, ExperimentSignificanceCode.SIGNIFICANT)