def setUp(self):
np.random.seed(41)
self.data, self.metadata = generate_random_data()
np.random.seed(41)
self.control_statistics = SampleStatistics(1000, 0.0, 1.0)
self.treatment_statistics = SampleStatistics(1200, 1.0, 1.0)
self.delta = 1.0
self.p = 0.04
self.statistical_power = 0.8
self.confidence_interval = {2.5: 0.1, 97.5: 1.1}
self.stop = True
self.corrected_p = 0.02
self.corrected_ci = {1.0: -0.7, 99.0: 0.7}
self.corrected_stop = False
self.simple_stats = SimpleTestStatistics(self.control_statistics,
self.treatment_statistics,
self.delta,
self.confidence_interval,
self.p,
self.statistical_power)
self.simple_stats_corrected = SimpleTestStatistics(self.control_statistics,
self.treatment_statistics,
self.delta,
self.corrected_ci,
self.corrected_p,
self.statistical_power)
self.es_stats = EarlyStoppingTestStatistics(self.control_statistics,
self.treatment_statistics,
self.delta,
self.confidence_interval,
self.p,
self.statistical_power,
self.stop)
self.es_stats_corrected = EarlyStoppingTestStatistics(self.control_statistics,
self.treatment_statistics,
self.delta,
self.corrected_ci,
self.corrected_p,
self.statistical_power,
self.corrected_stop)
variants = Variants('variant', 'control', 'treatment')
self.correction_method = CorrectionMethod.BONFERRONI
kpi = KPI('revenue')
self.statistical_test = StatisticalTest(self.data, kpi, [], variants)
self.statistical_test_result = StatisticalTestResult(self.statistical_test, self.simple_stats_corrected)
test_result1 = StatisticalTestResult(self.statistical_test,
CombinedTestStatistics(self.simple_stats, self.simple_stats_corrected))
test_result2 = StatisticalTestResult(self.statistical_test,
CombinedTestStatistics(self.es_stats, self.es_stats_corrected))
test_results = [test_result1, test_result2]
self.statistical_test_results = MultipleTestSuiteResult(test_results, self.correction_method)
def setUp(self):
"""
Load the needed datasets for all StatisticsTestCases and set the random
seed so that randomized algorithms show deterministic behaviour.
"""
# np.random.seed(0)
self.metrics, self.metadata = generate_random_data()
def get_two_multiple_test_suite_results():
""" Returns two multiple test suite results (for testing purposes of merge_with class method)
:return two multiple test suite results
:rtype MultipleTestSuiteResult, MultipleTestSuiteResult
"""
data, metadata = generate_random_data()
exp = Experiment(metadata)
kpi = KPI('normal_same')
variants = Variants('variant', 'B', 'A')
test_normal_same = StatisticalTest(data, kpi, [], variants)
derived_kpi = DerivedKPI('derived_kpi_one', 'normal_same',
'normal_shifted')
test_derived_kpi = StatisticalTest(data, derived_kpi, [], variants)
suite_with_normal_same = StatisticalTestSuite([test_normal_same],
CorrectionMethod.BONFERRONI)
suite_with_derived_kpi = StatisticalTestSuite([test_derived_kpi],
CorrectionMethod.BH)
mtsr_1 = exp.analyze_statistical_test_suite(suite_with_normal_same,
test_method='fixed_horizon')
mtsr_2 = exp.analyze_statistical_test_suite(suite_with_derived_kpi,
test_method='fixed_horizon')
return mtsr_1, mtsr_2
def test_direct_indexing(self):
A = ExperimentData(*generate_random_data())
# this should work
normal_shifted = A[['normal_shifted', 'feature']]
# this should not
with self.assertRaises(KeyError):
normal_shifted = A[['normal_shifted', 'non_existent_feature']]
def setUp(self):
np.random.seed(41)
self.data, self.metadata = generate_random_data()
# simple statistical test
self.test_kpi = KPI('normal_same')
self.test_variants = Variants('variant', 'A', 'B')
self.test_normal_same = StatisticalTest(self.test_kpi, [],
self.test_variants)
def test_data_generation(self):
df, md = generate_random_data()
A = ExperimentData(df, md)
self.assertIsNotNone(A.kpis)
self.assertIsNotNone(A.features)
self.assertIsNotNone(A.metadata)
# also test incomplete info again
with self.assertRaises(KeyError):
ExperimentData(df.drop('entity', axis=1), md)
def setUp(self):
np.random.seed(41)
self.data, self.metadata = generate_random_data()
# simple statistical test
self.test_kpi = KPI('normal_same')
self.test_variants = Variants('variant', 'A', 'B')
self.test_normal_same = StatisticalTest(self.data, self.test_kpi, [], self.test_variants)
# two multiple test suites results for merge_with tests
self.multiple_test_suite_result_1, self.multiple_test_suite_result_2 = get_two_multiple_test_suite_results()
def setUp(self):
"""
Load the needed datasets for all StatisticsTestCases and set the random
seed so that randomized algorithms show deterministic behaviour.
"""
np.random.seed(0)
data, metadata = generate_random_data()
self.column_names = list(set(data.columns) - set(['variant', 'entity']))
self.numeric_column_names = get_column_names_by_type(data, np.number)
self.data, self.metadata = data, metadata
def setUp(self):
np.random.seed(41)
self.data, self.metadata = generate_random_data()
np.random.seed(41)
self.control_statistics = SampleStatistics(1000, 0.0, 1.0)
self.treatment_statistics = SampleStatistics(1200, 1.0, 1.0)
self.delta = 1.0
self.p = 0.04
self.statistical_power = 0.8
self.confidence_interval = {2.5: 0.1, 97.5: 1.1}
self.stop = True
self.corrected_p = 0.02
self.corrected_ci = {1.0: -0.7, 99.0: 0.7}
self.corrected_stop = False
self.simple_stats = SimpleTestStatistics(
self.control_statistics, self.treatment_statistics, self.delta,
self.confidence_interval, self.p, self.statistical_power)
self.simple_stats_corrected = SimpleTestStatistics(
self.control_statistics, self.treatment_statistics, self.delta,
self.corrected_ci, self.corrected_p, self.statistical_power)
self.es_stats = EarlyStoppingTestStatistics(
self.control_statistics, self.treatment_statistics, self.delta,
self.confidence_interval, self.p, self.statistical_power,
self.stop)
self.es_stats_corrected = EarlyStoppingTestStatistics(
self.control_statistics, self.treatment_statistics, self.delta,
self.corrected_ci, self.corrected_p, self.statistical_power,
self.corrected_stop)
variants = Variants('variant', 'control', 'treatment')
self.correction_method = CorrectionMethod.BONFERRONI
kpi = KPI('revenue')
self.statistical_test = StatisticalTest(self.data, kpi, [], variants)
self.statistical_test_result = StatisticalTestResult(
self.statistical_test, self.simple_stats_corrected)
test_result1 = StatisticalTestResult(
self.statistical_test,
CombinedTestStatistics(self.simple_stats,
self.simple_stats_corrected))
test_result2 = StatisticalTestResult(
self.statistical_test,
CombinedTestStatistics(self.es_stats, self.es_stats_corrected))
test_results = [test_result1, test_result2]
self.statistical_test_results = MultipleTestSuiteResult(
test_results, self.correction_method)
def setUp(self):
np.random.seed(41)
self.data, self.metadata = generate_random_data()
# simple statistical test
self.test_kpi = KPI('normal_same')
self.test_variants = Variants('variant', 'A', 'B')
self.test_normal_same = StatisticalTest(self.data, self.test_kpi, [],
self.test_variants)
# two multiple test suites results for merge_with tests
self.multiple_test_suite_result_1, self.multiple_test_suite_result_2 = get_two_multiple_test_suite_results(
)
def setUp(self):
"""
Load the needed datasets for all TestCases and set the random
seed so that randomized algorithms show deterministic behaviour.
"""
np.random.seed(0)
self.data = Experiment('B', *generate_random_data())
# Create time column. TODO: Do this nicer
self.data.kpis['time_since_treatment'] = \
self.data.features['treatment_start_time']
# Make time part of index
self.data.kpis.set_index('time_since_treatment',
append=True,
inplace=True)
def setUp(self):
"""
Load the needed datasets for all StatisticsTestCases and set the random
seed so that randomized algorithms show deterministic behaviour.
"""
np.random.seed(0)
self.data = Experiment('B',
*generate_random_data(),
dbg=Dbg(dbg_lvl=5))
# Create time column. TODO: Do this nicer
self.data.kpis['time_since_treatment'] = \
self.data.features['treatment_start_time']
# Make time part of index
self.data.kpis.set_index('time_since_treatment',
append=True,
inplace=True)
# Metadata as generated by generate_random_data() for later checks
self.testmetadata = {
'primary_KPI': 'normal_shifted',
'source': 'simulated',
'experiment': 'random_data_generation'
}
def get_two_multiple_test_suite_results():
""" Returns two multiple test suite results (for testing purposes of merge_with class method)
:return two multiple test suite results
:rtype MultipleTestSuiteResult, MultipleTestSuiteResult
"""
data, metadata = generate_random_data()
exp = Experiment(metadata)
kpi = KPI('normal_same')
variants = Variants('variant', 'B', 'A')
test_normal_same = StatisticalTest(data, kpi, [], variants)
derived_kpi = DerivedKPI('derived_kpi_one', 'normal_same', 'normal_shifted')
test_derived_kpi = StatisticalTest(data, derived_kpi, [], variants)
suite_with_normal_same = StatisticalTestSuite([test_normal_same],
CorrectionMethod.BONFERRONI)
suite_with_derived_kpi = StatisticalTestSuite([test_derived_kpi], CorrectionMethod.BH)
mtsr_1 = exp.analyze_statistical_test_suite(suite_with_normal_same, test_method='fixed_horizon')
mtsr_2 = exp.analyze_statistical_test_suite(suite_with_derived_kpi, test_method='fixed_horizon')
return mtsr_1, mtsr_2
def test__trend__index_levels(self):
"""
Check if trend() returns the proper index levels
"""
np.random.seed(0)
metrics, metadata = generate_random_data()
metrics['time_since_treatment'] = metrics['treatment_start_time']
exp = Experiment('B', metrics, metadata, [4, 6])
# Perform sga()
result = exp.trend()
# Check if all index levels are present
index_levels = [
pd.Index([
u'normal_same', u'normal_shifted',
u'normal_shifted_by_feature', u'normal_unequal_variance'
],
dtype='object',
name=u'metric'),
pd.Index([u'-'], dtype='object', name=u'subgroup_metric'),
pd.Index([str(x) for x in np.arange(10.)],
dtype='object',
name=u'time'),
pd.Float64Index([], dtype='float64', name=u'subgroup'),
pd.Index(
[u'sample_size', u'uplift', u'uplift_pctile', u'variant_mean'],
dtype='object',
name=u'statistic'),
pd.Float64Index([2.5, 97.5], dtype='float64', name=u'pctile')
]
result_levels = list(result.df.index.levels)
# Check if all index levels match expectation TODO: Make nice
np.testing.assert_array_equal(index_levels[0], result_levels[0])
np.testing.assert_array_equal(index_levels[1], result_levels[1])
np.testing.assert_array_equal(index_levels[2], result_levels[2])
np.testing.assert_array_equal(index_levels[3], result_levels[3])
np.testing.assert_array_equal(index_levels[4], result_levels[4])
np.testing.assert_array_equal(index_levels[5], result_levels[5])
def test_initialize_without_kpi_with_feature(self):
"""Initialize ExperimentData with metrics=None, features=DF"""
metrics, _ = generate_random_data()
meta = {'source': 'simulated', 'experiment': 'random_data_generation'}
D = ExperimentData(None, meta, metrics)
# TODO: Check if this is the right approach
deltaWorker = statx.make_delta(assume_normal, percentiles, min_observations,
nruns, relative)
for kpi in kpi_subset:
for variant in variant_subset:
# TODO: Add metadata to res.metadata
res_obj = time_dependent_deltas(
self.kpis_time.reset_index()[['variant',
'time_since_treatment', kpi]],
variants=[variant, self.baseline_variant],
time_step=time_step,
cumulative=cumulative,
deltaWorker=deltaWorker)
res.df = pd.concat([res.df, res_obj.df])
# NB: assuming all binning objects based on the same feature are the same
res.set_binning(res_obj.binning)
# Return the result object
return res
if __name__ == '__main__':
from expan.core.util import generate_random_data
np.random.seed(0)
metrics, metadata = generate_random_data()
metrics['time_since_treatment'] = metrics['treatment_start_time']
metadata['estimatedSampleSize'] = 100000
exp = Experiment('B', metrics, metadata, [4, 6])
res = exp.delta(method='bayes_precision', kpi_subset=['normal_same'])
def setUp(self):
""" Load the needed data sets for all StatisticsTestCases and set the random
seed so that randomized algorithms show deterministic behaviour."""
np.random.seed(0)
data, metadata = generate_random_data()
self.data, self.metadata = data, metadata
feature_has = FeatureFilter('feature', 'has')
feature_non = FeatureFilter('feature', 'non')
feature_one = FeatureFilter('feature', 'feature that only has one data point')
# simple statistical test
self.kpi = KPI('normal_same')
self.variants = Variants('variant', 'B', 'A')
self.nonsense_variants = Variants('variant', 'C', 'D')
self.test_normal_same = StatisticalTest(self.data, self.kpi, [], self.variants)
self.test_nonsense_variant = StatisticalTest(self.data, self.kpi, [], self.nonsense_variants)
self.test_normal_same_feature_non = StatisticalTest(self.data, self.kpi, [feature_non], self.variants)
self.test_normal_same_feature_has = StatisticalTest(self.data, self.kpi, [feature_has], self.variants)
self.test_normal_same_feature_one = StatisticalTest(self.data, self.kpi, [feature_one], self.variants)
# statistical test with derived kpi
self.derived_kpi = DerivedKPI('derived_kpi_one', 'normal_same', 'normal_shifted')
self.test_derived_kpi = StatisticalTest(self.data, self.derived_kpi, [], self.variants)
# small dummy data frame
data_dummy = np.array([['index', 'entity', 'variant', 'normal_same', 'normal_shifted'],
[0, 1, 'A', 2.0, 1.0], [1, 2, 'B', 3.0, 2.0],
[2, 3, 'A', 3.0, 2.0], [3, 4, 'B', 2.5, 1.0]])
self.data_dummy_df = pd.DataFrame(data=data_dummy[1:, 1:],
columns=data_dummy[0, 1:]).convert_objects(convert_numeric=True)
data_dummy_with_nan = np.array([['index', 'entity', 'variant', 'normal_same', 'normal_shifted'],
[0, 1, 'A', 2.0, 1.0], [1, 2, 'B', 3.0, 2.0],
[2, 3, 'A', 3.0, 2.0], [3, 4, 'B', 2.5, 1.0],
[4, 5, 'A', 0.0, 0.0], [5, 6, 'B', None, None]])
self.data_dummy_df_with_nan = pd.DataFrame(data=data_dummy_with_nan[1:, 1:],
columns=data_dummy_with_nan[0, 1:]).convert_objects(convert_numeric=True)
# statistical test suite
self.suite_with_one_test = StatisticalTestSuite([self.test_normal_same])
self.suite_with_one_test_correction = StatisticalTestSuite([self.test_normal_same], CorrectionMethod.BH)
self.suite_with_two_tests = StatisticalTestSuite([self.test_normal_same, self.test_derived_kpi],
CorrectionMethod.BONFERRONI)
self.suite_with_one_subgroup = StatisticalTestSuite([self.test_normal_same_feature_one])
self.suite_with_three_subgroups = StatisticalTestSuite([self.test_normal_same_feature_non,
self.test_normal_same_feature_has,
self.test_normal_same_feature_one],
CorrectionMethod.BH)
# small dummy data frames producing zero std and -1 statistical power
data_dummy_zero_std = np.array([['index', 'entity', 'variant', 'normal_same', 'normal_shifted'],
[0, 1, 'A', 1.0, 1.0], [1, 2, 'B', 2.0, 2.0],
[2, 3, 'A', 1.0, 1.0], [3, 4, 'B', 2.0, 2.0]])
self.data_dummy_zero_std = pd.DataFrame(data=data_dummy_zero_std[1:, 1:],
columns=data_dummy_zero_std[0, 1:]).convert_objects(
convert_numeric=True)
self.test_normal_same_zero_std = StatisticalTest(self.data_dummy_zero_std, self.kpi, [], self.variants)
self.suite_with_one_test_zero_std = StatisticalTestSuite([self.test_normal_same_zero_std])
# small dummy data frames with all nan values
self.data_dummy_all_nan = pd.DataFrame({
'entity' : [1,2,3,4],
'variant' : ['A','B','C','D'],
'normal_same' : [np.nan] * 4,
'normal_shifted' : [np.nan] * 4,
})
self.test_normal_same_nan_data = StatisticalTest(self.data_dummy_all_nan, self.kpi, [], self.variants)
self.suite_with_one_test_with_nan_data = StatisticalTestSuite([self.test_normal_same_nan_data])
def test_initialize_with_metric_with_feature_df(self):
"""Initialize ExperimentData with metrics=DF, features=DF"""
metrics, meta = generate_random_data()
features = deepcopy(metrics)
D = ExperimentData(metrics, meta, features)
def test_initialize_with_metric_with_feature_list(self):
"""Initialize ExperimentData with metrics=DF, features=list"""
metrics, meta = generate_random_data()
D = ExperimentData(metrics, meta, [])
D = ExperimentData(metrics, meta, [4, 6])
from expan.core.util import generate_random_data
from expan.core.experiment import Experiment
from expan.core.statistical_test import KPI, Variants, StatisticalTest
data, metadata = generate_random_data()
print(data.head())
print(metadata)
kpi = KPI('normal_same')
variants = Variants(variant_column_name='variant',
control_name='B',
treatment_name='A')
test = StatisticalTest(data=data, kpi=kpi, features=[], variants=variants)
exp = Experiment(metadata=metadata)
result = exp.analyze_statistical_test(test)
print(result)
def setUp(self):
""" Load the needed data sets for all StatisticsTestCases and set the random
seed so that randomized algorithms show deterministic behaviour."""
np.random.seed(0)
data, metadata = generate_random_data()
self.data, self.metadata = data, metadata
feature_has = FeatureFilter('feature', 'has')
feature_non = FeatureFilter('feature', 'non')
feature_one = FeatureFilter('feature',
'feature that only has one data point')
# simple statistical test
self.kpi = KPI('normal_same')
self.variants = Variants('variant', 'B', 'A')
self.nonsense_variants = Variants('variant', 'C', 'D')
self.test_normal_same = StatisticalTest(self.data, self.kpi, [],
self.variants)
self.test_nonsense_variant = StatisticalTest(self.data, self.kpi, [],
self.nonsense_variants)
self.test_normal_same_feature_non = StatisticalTest(
self.data, self.kpi, [feature_non], self.variants)
self.test_normal_same_feature_has = StatisticalTest(
self.data, self.kpi, [feature_has], self.variants)
self.test_normal_same_feature_one = StatisticalTest(
self.data, self.kpi, [feature_one], self.variants)
# statistical test with derived kpi
self.derived_kpi = DerivedKPI('derived_kpi_one', 'normal_same',
'normal_shifted')
self.test_derived_kpi = StatisticalTest(self.data, self.derived_kpi,
[], self.variants)
# small dummy data frame
data_dummy = np.array(
[['index', 'entity', 'variant', 'normal_same', 'normal_shifted'],
[0, 1, 'A', 2.0, 1.0], [1, 2, 'B', 3.0, 2.0],
[2, 3, 'A', 3.0, 2.0], [3, 4, 'B', 2.5, 1.0]])
self.data_dummy_df = pd.DataFrame(
data=data_dummy[1:, 1:],
columns=data_dummy[0, 1:]).convert_objects(convert_numeric=True)
data_dummy_with_nan = np.array(
[['index', 'entity', 'variant', 'normal_same', 'normal_shifted'],
[0, 1, 'A', 2.0, 1.0], [1, 2, 'B', 3.0,
2.0], [2, 3, 'A', 3.0, 2.0],
[3, 4, 'B', 2.5, 1.0], [4, 5, 'A', 0.0, 0.0],
[5, 6, 'B', None, None]])
self.data_dummy_df_with_nan = pd.DataFrame(
data=data_dummy_with_nan[1:, 1:],
columns=data_dummy_with_nan[0, 1:]).convert_objects(
convert_numeric=True)
# statistical test suite
self.suite_with_one_test = StatisticalTestSuite(
[self.test_normal_same])
self.suite_with_one_test_correction = StatisticalTestSuite(
[self.test_normal_same], CorrectionMethod.BH)
self.suite_with_two_tests = StatisticalTestSuite(
[self.test_normal_same, self.test_derived_kpi],
CorrectionMethod.BONFERRONI)
self.suite_with_one_subgroup = StatisticalTestSuite(
[self.test_normal_same_feature_one])
self.suite_with_three_subgroups = StatisticalTestSuite([
self.test_normal_same_feature_non,
self.test_normal_same_feature_has,
self.test_normal_same_feature_one
], CorrectionMethod.BH)
# small dummy data frames producing zero std and -1 statistical power
data_dummy_zero_std = np.array(
[['index', 'entity', 'variant', 'normal_same', 'normal_shifted'],
[0, 1, 'A', 1.0, 1.0], [1, 2, 'B', 2.0, 2.0],
[2, 3, 'A', 1.0, 1.0], [3, 4, 'B', 2.0, 2.0]])
self.data_dummy_zero_std = pd.DataFrame(
data=data_dummy_zero_std[1:, 1:],
columns=data_dummy_zero_std[0, 1:]).convert_objects(
convert_numeric=True)
self.test_normal_same_zero_std = StatisticalTest(
self.data_dummy_zero_std, self.kpi, [], self.variants)
self.suite_with_one_test_zero_std = StatisticalTestSuite(
[self.test_normal_same_zero_std])
# small dummy data frames with all nan values
self.data_dummy_all_nan = pd.DataFrame({
'entity': [1, 2, 3, 4],
'variant': ['A', 'B', 'C', 'D'],
'normal_same': [np.nan] * 4,
'normal_shifted': [np.nan] * 4,
})
self.test_normal_same_nan_data = StatisticalTest(
self.data_dummy_all_nan, self.kpi, [], self.variants)
self.suite_with_one_test_with_nan_data = StatisticalTestSuite(
[self.test_normal_same_nan_data])
def test__trend__computation(self):
"""
Check if trend() functions properly
"""
np.random.seed(0)
metrics, metadata = generate_random_data()
metrics['time_since_treatment'] = metrics['treatment_start_time']
exp = Experiment('B', metrics, metadata, [4, 6])
# Perform sga() with non-cumulative results
result = exp.trend(cumulative=False)
# check uplift
df = result.statistic('trend', 'uplift', 'normal_shifted')
np.testing.assert_almost_equal(df.loc[:, ('value', 'A')],
np.array([
-1.009421, -0.847400, -1.119885,
-1.042597, -0.868819, -1.091165,
-0.952307, -1.028234, -0.978774,
-0.985696
]),
decimal=5)
# check pctile
df = result.statistic('trend', 'uplift_pctile', 'normal_shifted')
np.testing.assert_almost_equal(
df.loc[:, ('value', 'A')],
np.array([
-1.137482, -0.881360, -0.970678, -0.724122, -1.245795,
-0.993975, -1.178494, -0.906699, -0.993683, -0.743954,
-1.225361, -0.956969, -1.082180, -0.822435, -1.151715,
-0.904753, -1.095209, -0.862340, -1.109407, -0.861985
]),
decimal=5)
# check samplesize
df = result.statistic('trend', 'sample_size', 'normal_shifted')
np.testing.assert_almost_equal(
df.loc[:, 'value'],
np.column_stack(
([649, 595, 600, 590, 625, 602, 607, 608, 616,
616], [405, 401, 378, 362, 377, 369, 406, 392, 414, 388])),
decimal=5)
# check variant_mean
df = result.statistic('trend', 'variant_mean', 'normal_shifted')
np.testing.assert_almost_equal(df.loc[:, 'value'],
np.column_stack(([
0.005761, 0.057487, -0.067107,
0.001125, 0.093085, -0.067894,
-0.030500, -0.060996, 0.016257,
-0.006091
], [
1.015182, 0.904887, 1.052778,
1.043721, 0.961904, 1.023271,
0.921807, 0.967238, 0.995031,
0.979605
])),
decimal=5)
# Perform sga() with cumulative results
result = exp.trend()
# check uplift
df = result.statistic('trend', 'uplift', 'normal_shifted')
np.testing.assert_almost_equal(df.loc[:, ('value', 'A')],
np.array([
-1.009421, -0.929807, -0.991088,
-1.003129, -0.976023, -0.994857,
-0.988167, -0.993119, -0.991571,
-0.990986
]),
decimal=5)
# check pctile
df = result.statistic('trend', 'uplift_pctile', 'normal_shifted')
np.testing.assert_almost_equal(
df.loc[:, ('value', 'A')],
np.array([
-1.137482, -0.881360, -1.018794, -0.840820, -1.063820,
-0.918356, -1.067283, -0.938976, -1.033110, -0.918936,
-1.047413, -0.942302, -1.036888, -0.939446, -1.038455,
-0.947784, -1.033861, -0.949280, -1.031002, -0.950970
]),
decimal=5)
# check samplesize
df = result.statistic('trend', 'sample_size', 'normal_shifted')
np.testing.assert_almost_equal(
df.loc[:, 'value'],
np.column_stack(
([649, 1244, 1844, 2434, 3059, 3661, 4268, 4876, 5492, 6108],
[405, 806, 1184, 1546, 1923, 2292, 2698, 3090, 3504, 3892])),
decimal=5)
# check variant_mean
df = result.statistic('trend', 'variant_mean', 'normal_shifted')
np.testing.assert_almost_equal(df.loc[:, 'value'],
np.column_stack(([
0.005761, 0.030501, -0.001258,
-0.000681, 0.018477, 0.004274,
-0.000671, -0.008193, -0.005451,
-0.005515
], [
1.015182, 0.960308, 0.989830,
1.002449, 0.994500, 0.999132,
0.987496, 0.984926, 0.986120,
0.985470
])),
decimal=5)
# check metadata is preserved
np.testing.assert_equal(
True,
all(item in result.metadata.items()
for item in self.testmetadata.items()))
json_object = json.loads(
self.data.sga(percentiles=[2.5, 5.0, 95.0, 97.5]).to_json())
self.assertEqual(2, len(json_object['variants']))
self.assertEqual(4, len(json_object['variants'][0]['metrics']))
self.assertEqual(
2,
len(json_object['variants'][0]['metrics'][0]['subgroup_metrics']))
self.assertGreaterEqual(
4,
len(json_object['variants'][0]['metrics'][0]['subgroup_metrics'][0]
['subgroups']))
self.assertEqual(
4,
len(json_object['variants'][0]['metrics'][0]['subgroup_metrics'][0]
['subgroups'][0]['statistics']))
self.assertEqual(
1,
len(json_object['variants'][0]['metrics'][0]['subgroup_metrics'][0]
['subgroups'][0]['statistics'][3]['pctiles']))
def test_to_json_trend(self):
# to_json() doesn't handle trend() results yet!
self.assertIsNone(self.data.trend().to_json())
if __name__ == '__main__':
# unittest.main()
np.random.seed(0)
exp = Experiment('B', *generate_random_data())
res = exp.delta(['normal_shifted'])
