def setUp(self): np.random.seed(41) self.data, self.metadata = generate_random_data() np.random.seed(41) self.control_statistics = SampleStatistics(1000, 0.0, 1.0) self.treatment_statistics = SampleStatistics(1200, 1.0, 1.0) self.delta = 1.0 self.p = 0.04 self.statistical_power = 0.8 self.confidence_interval = {2.5: 0.1, 97.5: 1.1} self.stop = True self.corrected_p = 0.02 self.corrected_ci = {1.0: -0.7, 99.0: 0.7} self.corrected_stop = False self.simple_stats = SimpleTestStatistics(self.control_statistics, self.treatment_statistics, self.delta, self.confidence_interval, self.p, self.statistical_power) self.simple_stats_corrected = SimpleTestStatistics(self.control_statistics, self.treatment_statistics, self.delta, self.corrected_ci, self.corrected_p, self.statistical_power) self.es_stats = EarlyStoppingTestStatistics(self.control_statistics, self.treatment_statistics, self.delta, self.confidence_interval, self.p, self.statistical_power, self.stop) self.es_stats_corrected = EarlyStoppingTestStatistics(self.control_statistics, self.treatment_statistics, self.delta, self.corrected_ci, self.corrected_p, self.statistical_power, self.corrected_stop) variants = Variants('variant', 'control', 'treatment') self.correction_method = CorrectionMethod.BONFERRONI kpi = KPI('revenue') self.statistical_test = StatisticalTest(self.data, kpi, [], variants) self.statistical_test_result = StatisticalTestResult(self.statistical_test, self.simple_stats_corrected) test_result1 = StatisticalTestResult(self.statistical_test, CombinedTestStatistics(self.simple_stats, self.simple_stats_corrected)) test_result2 = StatisticalTestResult(self.statistical_test, CombinedTestStatistics(self.es_stats, self.es_stats_corrected)) test_results = [test_result1, test_result2] self.statistical_test_results = MultipleTestSuiteResult(test_results, self.correction_method)
def setUp(self): """ Load the needed datasets for all StatisticsTestCases and set the random seed so that randomized algorithms show deterministic behaviour. """ # np.random.seed(0) self.metrics, self.metadata = generate_random_data()
def get_two_multiple_test_suite_results(): """ Returns two multiple test suite results (for testing purposes of merge_with class method) :return two multiple test suite results :rtype MultipleTestSuiteResult, MultipleTestSuiteResult """ data, metadata = generate_random_data() exp = Experiment(metadata) kpi = KPI('normal_same') variants = Variants('variant', 'B', 'A') test_normal_same = StatisticalTest(data, kpi, [], variants) derived_kpi = DerivedKPI('derived_kpi_one', 'normal_same', 'normal_shifted') test_derived_kpi = StatisticalTest(data, derived_kpi, [], variants) suite_with_normal_same = StatisticalTestSuite([test_normal_same], CorrectionMethod.BONFERRONI) suite_with_derived_kpi = StatisticalTestSuite([test_derived_kpi], CorrectionMethod.BH) mtsr_1 = exp.analyze_statistical_test_suite(suite_with_normal_same, test_method='fixed_horizon') mtsr_2 = exp.analyze_statistical_test_suite(suite_with_derived_kpi, test_method='fixed_horizon') return mtsr_1, mtsr_2
def test_direct_indexing(self): A = ExperimentData(*generate_random_data()) # this should work normal_shifted = A[['normal_shifted', 'feature']] # this should not with self.assertRaises(KeyError): normal_shifted = A[['normal_shifted', 'non_existent_feature']]
def setUp(self): np.random.seed(41) self.data, self.metadata = generate_random_data() # simple statistical test self.test_kpi = KPI('normal_same') self.test_variants = Variants('variant', 'A', 'B') self.test_normal_same = StatisticalTest(self.test_kpi, [], self.test_variants)
def test_data_generation(self): df, md = generate_random_data() A = ExperimentData(df, md) self.assertIsNotNone(A.kpis) self.assertIsNotNone(A.features) self.assertIsNotNone(A.metadata) # also test incomplete info again with self.assertRaises(KeyError): ExperimentData(df.drop('entity', axis=1), md)
def setUp(self): np.random.seed(41) self.data, self.metadata = generate_random_data() # simple statistical test self.test_kpi = KPI('normal_same') self.test_variants = Variants('variant', 'A', 'B') self.test_normal_same = StatisticalTest(self.data, self.test_kpi, [], self.test_variants) # two multiple test suites results for merge_with tests self.multiple_test_suite_result_1, self.multiple_test_suite_result_2 = get_two_multiple_test_suite_results()
def setUp(self): """ Load the needed datasets for all StatisticsTestCases and set the random seed so that randomized algorithms show deterministic behaviour. """ np.random.seed(0) data, metadata = generate_random_data() self.column_names = list(set(data.columns) - set(['variant', 'entity'])) self.numeric_column_names = get_column_names_by_type(data, np.number) self.data, self.metadata = data, metadata
def setUp(self): np.random.seed(41) self.data, self.metadata = generate_random_data() np.random.seed(41) self.control_statistics = SampleStatistics(1000, 0.0, 1.0) self.treatment_statistics = SampleStatistics(1200, 1.0, 1.0) self.delta = 1.0 self.p = 0.04 self.statistical_power = 0.8 self.confidence_interval = {2.5: 0.1, 97.5: 1.1} self.stop = True self.corrected_p = 0.02 self.corrected_ci = {1.0: -0.7, 99.0: 0.7} self.corrected_stop = False self.simple_stats = SimpleTestStatistics( self.control_statistics, self.treatment_statistics, self.delta, self.confidence_interval, self.p, self.statistical_power) self.simple_stats_corrected = SimpleTestStatistics( self.control_statistics, self.treatment_statistics, self.delta, self.corrected_ci, self.corrected_p, self.statistical_power) self.es_stats = EarlyStoppingTestStatistics( self.control_statistics, self.treatment_statistics, self.delta, self.confidence_interval, self.p, self.statistical_power, self.stop) self.es_stats_corrected = EarlyStoppingTestStatistics( self.control_statistics, self.treatment_statistics, self.delta, self.corrected_ci, self.corrected_p, self.statistical_power, self.corrected_stop) variants = Variants('variant', 'control', 'treatment') self.correction_method = CorrectionMethod.BONFERRONI kpi = KPI('revenue') self.statistical_test = StatisticalTest(self.data, kpi, [], variants) self.statistical_test_result = StatisticalTestResult( self.statistical_test, self.simple_stats_corrected) test_result1 = StatisticalTestResult( self.statistical_test, CombinedTestStatistics(self.simple_stats, self.simple_stats_corrected)) test_result2 = StatisticalTestResult( self.statistical_test, CombinedTestStatistics(self.es_stats, self.es_stats_corrected)) test_results = [test_result1, test_result2] self.statistical_test_results = MultipleTestSuiteResult( test_results, self.correction_method)
def setUp(self): np.random.seed(41) self.data, self.metadata = generate_random_data() # simple statistical test self.test_kpi = KPI('normal_same') self.test_variants = Variants('variant', 'A', 'B') self.test_normal_same = StatisticalTest(self.data, self.test_kpi, [], self.test_variants) # two multiple test suites results for merge_with tests self.multiple_test_suite_result_1, self.multiple_test_suite_result_2 = get_two_multiple_test_suite_results( )
def setUp(self): """ Load the needed datasets for all TestCases and set the random seed so that randomized algorithms show deterministic behaviour. """ np.random.seed(0) self.data = Experiment('B', *generate_random_data()) # Create time column. TODO: Do this nicer self.data.kpis['time_since_treatment'] = \ self.data.features['treatment_start_time'] # Make time part of index self.data.kpis.set_index('time_since_treatment', append=True, inplace=True)
def setUp(self): """ Load the needed datasets for all StatisticsTestCases and set the random seed so that randomized algorithms show deterministic behaviour. """ np.random.seed(0) self.data = Experiment('B', *generate_random_data(), dbg=Dbg(dbg_lvl=5)) # Create time column. TODO: Do this nicer self.data.kpis['time_since_treatment'] = \ self.data.features['treatment_start_time'] # Make time part of index self.data.kpis.set_index('time_since_treatment', append=True, inplace=True) # Metadata as generated by generate_random_data() for later checks self.testmetadata = { 'primary_KPI': 'normal_shifted', 'source': 'simulated', 'experiment': 'random_data_generation' }
def get_two_multiple_test_suite_results(): """ Returns two multiple test suite results (for testing purposes of merge_with class method) :return two multiple test suite results :rtype MultipleTestSuiteResult, MultipleTestSuiteResult """ data, metadata = generate_random_data() exp = Experiment(metadata) kpi = KPI('normal_same') variants = Variants('variant', 'B', 'A') test_normal_same = StatisticalTest(data, kpi, [], variants) derived_kpi = DerivedKPI('derived_kpi_one', 'normal_same', 'normal_shifted') test_derived_kpi = StatisticalTest(data, derived_kpi, [], variants) suite_with_normal_same = StatisticalTestSuite([test_normal_same], CorrectionMethod.BONFERRONI) suite_with_derived_kpi = StatisticalTestSuite([test_derived_kpi], CorrectionMethod.BH) mtsr_1 = exp.analyze_statistical_test_suite(suite_with_normal_same, test_method='fixed_horizon') mtsr_2 = exp.analyze_statistical_test_suite(suite_with_derived_kpi, test_method='fixed_horizon') return mtsr_1, mtsr_2
def test__trend__index_levels(self): """ Check if trend() returns the proper index levels """ np.random.seed(0) metrics, metadata = generate_random_data() metrics['time_since_treatment'] = metrics['treatment_start_time'] exp = Experiment('B', metrics, metadata, [4, 6]) # Perform sga() result = exp.trend() # Check if all index levels are present index_levels = [ pd.Index([ u'normal_same', u'normal_shifted', u'normal_shifted_by_feature', u'normal_unequal_variance' ], dtype='object', name=u'metric'), pd.Index([u'-'], dtype='object', name=u'subgroup_metric'), pd.Index([str(x) for x in np.arange(10.)], dtype='object', name=u'time'), pd.Float64Index([], dtype='float64', name=u'subgroup'), pd.Index( [u'sample_size', u'uplift', u'uplift_pctile', u'variant_mean'], dtype='object', name=u'statistic'), pd.Float64Index([2.5, 97.5], dtype='float64', name=u'pctile') ] result_levels = list(result.df.index.levels) # Check if all index levels match expectation TODO: Make nice np.testing.assert_array_equal(index_levels[0], result_levels[0]) np.testing.assert_array_equal(index_levels[1], result_levels[1]) np.testing.assert_array_equal(index_levels[2], result_levels[2]) np.testing.assert_array_equal(index_levels[3], result_levels[3]) np.testing.assert_array_equal(index_levels[4], result_levels[4]) np.testing.assert_array_equal(index_levels[5], result_levels[5])
def test_initialize_without_kpi_with_feature(self): """Initialize ExperimentData with metrics=None, features=DF""" metrics, _ = generate_random_data() meta = {'source': 'simulated', 'experiment': 'random_data_generation'} D = ExperimentData(None, meta, metrics)
# TODO: Check if this is the right approach deltaWorker = statx.make_delta(assume_normal, percentiles, min_observations, nruns, relative) for kpi in kpi_subset: for variant in variant_subset: # TODO: Add metadata to res.metadata res_obj = time_dependent_deltas( self.kpis_time.reset_index()[['variant', 'time_since_treatment', kpi]], variants=[variant, self.baseline_variant], time_step=time_step, cumulative=cumulative, deltaWorker=deltaWorker) res.df = pd.concat([res.df, res_obj.df]) # NB: assuming all binning objects based on the same feature are the same res.set_binning(res_obj.binning) # Return the result object return res if __name__ == '__main__': from expan.core.util import generate_random_data np.random.seed(0) metrics, metadata = generate_random_data() metrics['time_since_treatment'] = metrics['treatment_start_time'] metadata['estimatedSampleSize'] = 100000 exp = Experiment('B', metrics, metadata, [4, 6]) res = exp.delta(method='bayes_precision', kpi_subset=['normal_same'])
def setUp(self): """ Load the needed data sets for all StatisticsTestCases and set the random seed so that randomized algorithms show deterministic behaviour.""" np.random.seed(0) data, metadata = generate_random_data() self.data, self.metadata = data, metadata feature_has = FeatureFilter('feature', 'has') feature_non = FeatureFilter('feature', 'non') feature_one = FeatureFilter('feature', 'feature that only has one data point') # simple statistical test self.kpi = KPI('normal_same') self.variants = Variants('variant', 'B', 'A') self.nonsense_variants = Variants('variant', 'C', 'D') self.test_normal_same = StatisticalTest(self.data, self.kpi, [], self.variants) self.test_nonsense_variant = StatisticalTest(self.data, self.kpi, [], self.nonsense_variants) self.test_normal_same_feature_non = StatisticalTest(self.data, self.kpi, [feature_non], self.variants) self.test_normal_same_feature_has = StatisticalTest(self.data, self.kpi, [feature_has], self.variants) self.test_normal_same_feature_one = StatisticalTest(self.data, self.kpi, [feature_one], self.variants) # statistical test with derived kpi self.derived_kpi = DerivedKPI('derived_kpi_one', 'normal_same', 'normal_shifted') self.test_derived_kpi = StatisticalTest(self.data, self.derived_kpi, [], self.variants) # small dummy data frame data_dummy = np.array([['index', 'entity', 'variant', 'normal_same', 'normal_shifted'], [0, 1, 'A', 2.0, 1.0], [1, 2, 'B', 3.0, 2.0], [2, 3, 'A', 3.0, 2.0], [3, 4, 'B', 2.5, 1.0]]) self.data_dummy_df = pd.DataFrame(data=data_dummy[1:, 1:], columns=data_dummy[0, 1:]).convert_objects(convert_numeric=True) data_dummy_with_nan = np.array([['index', 'entity', 'variant', 'normal_same', 'normal_shifted'], [0, 1, 'A', 2.0, 1.0], [1, 2, 'B', 3.0, 2.0], [2, 3, 'A', 3.0, 2.0], [3, 4, 'B', 2.5, 1.0], [4, 5, 'A', 0.0, 0.0], [5, 6, 'B', None, None]]) self.data_dummy_df_with_nan = pd.DataFrame(data=data_dummy_with_nan[1:, 1:], columns=data_dummy_with_nan[0, 1:]).convert_objects(convert_numeric=True) # statistical test suite self.suite_with_one_test = StatisticalTestSuite([self.test_normal_same]) self.suite_with_one_test_correction = StatisticalTestSuite([self.test_normal_same], CorrectionMethod.BH) self.suite_with_two_tests = StatisticalTestSuite([self.test_normal_same, self.test_derived_kpi], CorrectionMethod.BONFERRONI) self.suite_with_one_subgroup = StatisticalTestSuite([self.test_normal_same_feature_one]) self.suite_with_three_subgroups = StatisticalTestSuite([self.test_normal_same_feature_non, self.test_normal_same_feature_has, self.test_normal_same_feature_one], CorrectionMethod.BH) # small dummy data frames producing zero std and -1 statistical power data_dummy_zero_std = np.array([['index', 'entity', 'variant', 'normal_same', 'normal_shifted'], [0, 1, 'A', 1.0, 1.0], [1, 2, 'B', 2.0, 2.0], [2, 3, 'A', 1.0, 1.0], [3, 4, 'B', 2.0, 2.0]]) self.data_dummy_zero_std = pd.DataFrame(data=data_dummy_zero_std[1:, 1:], columns=data_dummy_zero_std[0, 1:]).convert_objects( convert_numeric=True) self.test_normal_same_zero_std = StatisticalTest(self.data_dummy_zero_std, self.kpi, [], self.variants) self.suite_with_one_test_zero_std = StatisticalTestSuite([self.test_normal_same_zero_std]) # small dummy data frames with all nan values self.data_dummy_all_nan = pd.DataFrame({ 'entity' : [1,2,3,4], 'variant' : ['A','B','C','D'], 'normal_same' : [np.nan] * 4, 'normal_shifted' : [np.nan] * 4, }) self.test_normal_same_nan_data = StatisticalTest(self.data_dummy_all_nan, self.kpi, [], self.variants) self.suite_with_one_test_with_nan_data = StatisticalTestSuite([self.test_normal_same_nan_data])
def test_initialize_with_metric_with_feature_df(self): """Initialize ExperimentData with metrics=DF, features=DF""" metrics, meta = generate_random_data() features = deepcopy(metrics) D = ExperimentData(metrics, meta, features)
def test_initialize_with_metric_with_feature_list(self): """Initialize ExperimentData with metrics=DF, features=list""" metrics, meta = generate_random_data() D = ExperimentData(metrics, meta, []) D = ExperimentData(metrics, meta, [4, 6])
from expan.core.util import generate_random_data from expan.core.experiment import Experiment from expan.core.statistical_test import KPI, Variants, StatisticalTest data, metadata = generate_random_data() print(data.head()) print(metadata) kpi = KPI('normal_same') variants = Variants(variant_column_name='variant', control_name='B', treatment_name='A') test = StatisticalTest(data=data, kpi=kpi, features=[], variants=variants) exp = Experiment(metadata=metadata) result = exp.analyze_statistical_test(test) print(result)
def setUp(self): """ Load the needed data sets for all StatisticsTestCases and set the random seed so that randomized algorithms show deterministic behaviour.""" np.random.seed(0) data, metadata = generate_random_data() self.data, self.metadata = data, metadata feature_has = FeatureFilter('feature', 'has') feature_non = FeatureFilter('feature', 'non') feature_one = FeatureFilter('feature', 'feature that only has one data point') # simple statistical test self.kpi = KPI('normal_same') self.variants = Variants('variant', 'B', 'A') self.nonsense_variants = Variants('variant', 'C', 'D') self.test_normal_same = StatisticalTest(self.data, self.kpi, [], self.variants) self.test_nonsense_variant = StatisticalTest(self.data, self.kpi, [], self.nonsense_variants) self.test_normal_same_feature_non = StatisticalTest( self.data, self.kpi, [feature_non], self.variants) self.test_normal_same_feature_has = StatisticalTest( self.data, self.kpi, [feature_has], self.variants) self.test_normal_same_feature_one = StatisticalTest( self.data, self.kpi, [feature_one], self.variants) # statistical test with derived kpi self.derived_kpi = DerivedKPI('derived_kpi_one', 'normal_same', 'normal_shifted') self.test_derived_kpi = StatisticalTest(self.data, self.derived_kpi, [], self.variants) # small dummy data frame data_dummy = np.array( [['index', 'entity', 'variant', 'normal_same', 'normal_shifted'], [0, 1, 'A', 2.0, 1.0], [1, 2, 'B', 3.0, 2.0], [2, 3, 'A', 3.0, 2.0], [3, 4, 'B', 2.5, 1.0]]) self.data_dummy_df = pd.DataFrame( data=data_dummy[1:, 1:], columns=data_dummy[0, 1:]).convert_objects(convert_numeric=True) data_dummy_with_nan = np.array( [['index', 'entity', 'variant', 'normal_same', 'normal_shifted'], [0, 1, 'A', 2.0, 1.0], [1, 2, 'B', 3.0, 2.0], [2, 3, 'A', 3.0, 2.0], [3, 4, 'B', 2.5, 1.0], [4, 5, 'A', 0.0, 0.0], [5, 6, 'B', None, None]]) self.data_dummy_df_with_nan = pd.DataFrame( data=data_dummy_with_nan[1:, 1:], columns=data_dummy_with_nan[0, 1:]).convert_objects( convert_numeric=True) # statistical test suite self.suite_with_one_test = StatisticalTestSuite( [self.test_normal_same]) self.suite_with_one_test_correction = StatisticalTestSuite( [self.test_normal_same], CorrectionMethod.BH) self.suite_with_two_tests = StatisticalTestSuite( [self.test_normal_same, self.test_derived_kpi], CorrectionMethod.BONFERRONI) self.suite_with_one_subgroup = StatisticalTestSuite( [self.test_normal_same_feature_one]) self.suite_with_three_subgroups = StatisticalTestSuite([ self.test_normal_same_feature_non, self.test_normal_same_feature_has, self.test_normal_same_feature_one ], CorrectionMethod.BH) # small dummy data frames producing zero std and -1 statistical power data_dummy_zero_std = np.array( [['index', 'entity', 'variant', 'normal_same', 'normal_shifted'], [0, 1, 'A', 1.0, 1.0], [1, 2, 'B', 2.0, 2.0], [2, 3, 'A', 1.0, 1.0], [3, 4, 'B', 2.0, 2.0]]) self.data_dummy_zero_std = pd.DataFrame( data=data_dummy_zero_std[1:, 1:], columns=data_dummy_zero_std[0, 1:]).convert_objects( convert_numeric=True) self.test_normal_same_zero_std = StatisticalTest( self.data_dummy_zero_std, self.kpi, [], self.variants) self.suite_with_one_test_zero_std = StatisticalTestSuite( [self.test_normal_same_zero_std]) # small dummy data frames with all nan values self.data_dummy_all_nan = pd.DataFrame({ 'entity': [1, 2, 3, 4], 'variant': ['A', 'B', 'C', 'D'], 'normal_same': [np.nan] * 4, 'normal_shifted': [np.nan] * 4, }) self.test_normal_same_nan_data = StatisticalTest( self.data_dummy_all_nan, self.kpi, [], self.variants) self.suite_with_one_test_with_nan_data = StatisticalTestSuite( [self.test_normal_same_nan_data])
def test__trend__computation(self): """ Check if trend() functions properly """ np.random.seed(0) metrics, metadata = generate_random_data() metrics['time_since_treatment'] = metrics['treatment_start_time'] exp = Experiment('B', metrics, metadata, [4, 6]) # Perform sga() with non-cumulative results result = exp.trend(cumulative=False) # check uplift df = result.statistic('trend', 'uplift', 'normal_shifted') np.testing.assert_almost_equal(df.loc[:, ('value', 'A')], np.array([ -1.009421, -0.847400, -1.119885, -1.042597, -0.868819, -1.091165, -0.952307, -1.028234, -0.978774, -0.985696 ]), decimal=5) # check pctile df = result.statistic('trend', 'uplift_pctile', 'normal_shifted') np.testing.assert_almost_equal( df.loc[:, ('value', 'A')], np.array([ -1.137482, -0.881360, -0.970678, -0.724122, -1.245795, -0.993975, -1.178494, -0.906699, -0.993683, -0.743954, -1.225361, -0.956969, -1.082180, -0.822435, -1.151715, -0.904753, -1.095209, -0.862340, -1.109407, -0.861985 ]), decimal=5) # check samplesize df = result.statistic('trend', 'sample_size', 'normal_shifted') np.testing.assert_almost_equal( df.loc[:, 'value'], np.column_stack( ([649, 595, 600, 590, 625, 602, 607, 608, 616, 616], [405, 401, 378, 362, 377, 369, 406, 392, 414, 388])), decimal=5) # check variant_mean df = result.statistic('trend', 'variant_mean', 'normal_shifted') np.testing.assert_almost_equal(df.loc[:, 'value'], np.column_stack(([ 0.005761, 0.057487, -0.067107, 0.001125, 0.093085, -0.067894, -0.030500, -0.060996, 0.016257, -0.006091 ], [ 1.015182, 0.904887, 1.052778, 1.043721, 0.961904, 1.023271, 0.921807, 0.967238, 0.995031, 0.979605 ])), decimal=5) # Perform sga() with cumulative results result = exp.trend() # check uplift df = result.statistic('trend', 'uplift', 'normal_shifted') np.testing.assert_almost_equal(df.loc[:, ('value', 'A')], np.array([ -1.009421, -0.929807, -0.991088, -1.003129, -0.976023, -0.994857, -0.988167, -0.993119, -0.991571, -0.990986 ]), decimal=5) # check pctile df = result.statistic('trend', 'uplift_pctile', 'normal_shifted') np.testing.assert_almost_equal( df.loc[:, ('value', 'A')], np.array([ -1.137482, -0.881360, -1.018794, -0.840820, -1.063820, -0.918356, -1.067283, -0.938976, -1.033110, -0.918936, -1.047413, -0.942302, -1.036888, -0.939446, -1.038455, -0.947784, -1.033861, -0.949280, -1.031002, -0.950970 ]), decimal=5) # check samplesize df = result.statistic('trend', 'sample_size', 'normal_shifted') np.testing.assert_almost_equal( df.loc[:, 'value'], np.column_stack( ([649, 1244, 1844, 2434, 3059, 3661, 4268, 4876, 5492, 6108], [405, 806, 1184, 1546, 1923, 2292, 2698, 3090, 3504, 3892])), decimal=5) # check variant_mean df = result.statistic('trend', 'variant_mean', 'normal_shifted') np.testing.assert_almost_equal(df.loc[:, 'value'], np.column_stack(([ 0.005761, 0.030501, -0.001258, -0.000681, 0.018477, 0.004274, -0.000671, -0.008193, -0.005451, -0.005515 ], [ 1.015182, 0.960308, 0.989830, 1.002449, 0.994500, 0.999132, 0.987496, 0.984926, 0.986120, 0.985470 ])), decimal=5) # check metadata is preserved np.testing.assert_equal( True, all(item in result.metadata.items() for item in self.testmetadata.items()))
json_object = json.loads( self.data.sga(percentiles=[2.5, 5.0, 95.0, 97.5]).to_json()) self.assertEqual(2, len(json_object['variants'])) self.assertEqual(4, len(json_object['variants'][0]['metrics'])) self.assertEqual( 2, len(json_object['variants'][0]['metrics'][0]['subgroup_metrics'])) self.assertGreaterEqual( 4, len(json_object['variants'][0]['metrics'][0]['subgroup_metrics'][0] ['subgroups'])) self.assertEqual( 4, len(json_object['variants'][0]['metrics'][0]['subgroup_metrics'][0] ['subgroups'][0]['statistics'])) self.assertEqual( 1, len(json_object['variants'][0]['metrics'][0]['subgroup_metrics'][0] ['subgroups'][0]['statistics'][3]['pctiles'])) def test_to_json_trend(self): # to_json() doesn't handle trend() results yet! self.assertIsNone(self.data.trend().to_json()) if __name__ == '__main__': # unittest.main() np.random.seed(0) exp = Experiment('B', *generate_random_data()) res = exp.delta(['normal_shifted'])