def test_initialize_without_kpi_without_feature(self):
"""Initialize ExperimentData with metrics=None, features=[]/'default'"""
meta = {'source': 'simulated', 'experiment': 'random_data_generation'}
with self.assertRaises(ValueError):
D = ExperimentData(None, meta, [])
with self.assertRaises(ValueError):
D = ExperimentData(None, meta, 'default')
def test_data_generation(self):
df, md = generate_random_data()
A = ExperimentData(df, md)
self.assertIsNotNone(A.kpis)
self.assertIsNotNone(A.features)
self.assertIsNotNone(A.metadata)
# also test incomplete info again
with self.assertRaises(KeyError):
ExperimentData(df.drop('entity', axis=1), md)
def test_outlier_filtering_n_filtered(self):
"""Check if the number of filtered entities is persisted in the metadata"""
D = ExperimentData(self.metrics, self.metadata, 'default')
D.filter_outliers(rules=[{
"metric": "normal_shifted",
"type": "threshold",
"value": -1.0,
"kind": "lower"
}])
self.assertEqual(D.metadata['n_filtered'], [1082])
def run_analysis(features_file, kpis_file, metadata_file):
"""
Load kpis and features from file and pass them to expan to perform delta and subgroup analyses
Args:
features_file: features file path
kpis_file: kpis file path
metadata_file: metadata file path
Returns:
delta analysis results and subgroup analysis results as a tuple
"""
kpis = pd.read_csv(kpis_file)
if features_file:
features = pd.read_csv(features_file)
else:
features = 'default'
print(features)
metadata = parse_metadata(metadata_file)
exp_data = ExperimentData(metrics=kpis,
metadata=metadata,
features=features)
exp = Experiment(baseline_variant=metadata['baseline_variant'],
metrics_or_kpis=kpis,
metadata=metadata,
features=features)
return (exp.delta(), exp.sga())
def test_direct_indexing(self):
A = ExperimentData(*generate_random_data())
# this should work
normal_shifted = A[['normal_shifted', 'feature']]
# this should not
with self.assertRaises(KeyError):
normal_shifted = A[['normal_shifted', 'non_existent_feature']]
def test_initialize_without_kpi_with_feature(self):
"""Initialize ExperimentData with metrics=None, features=DF"""
metrics, _ = generate_random_data()
meta = {
'source': 'simulated',
'experiment': 'random_data_generation'
}
D = ExperimentData(None, meta, metrics)
def test_init_with_time_resolved_kpi(self): """Initialize ExperimentData with time-resolved KPI data""" n = 5 # replicate raw data and create synthetic time domain metrics_time = self.metrics.loc[np.repeat(self.metrics.index.values, n)] metrics_time.loc[:, 'time_since_treatment'] = np.tile(range(5), self.metrics.shape[0]) D = ExperimentData(metrics_time, self.metadata, [4]) self.assertIsNotNone(D.kpis_time) self.assertEquals(D.kpis.shape[0] * n, D.kpis_time.shape[0])
def test_outlier_filtering_no_treatment_start(self):
"""Check if outlier filtering issues a warning when treatment_start_time is not available"""
# initialize with only the kpi data
D = ExperimentData(
self.metrics[['entity', 'variant', 'normal_shifted']],
self.metadata, 'default')
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
# Trigger a warning.
D.filter_outliers(rules=[{
"metric": "normal_shifted",
"type": "threshold",
"value": -1.0,
"kind": "lower",
"time_interval": 30758400,
"treatment_stop_time": 30758500
}])
# Verify warning exists
assert len(w) == 1
def run_analysis(features_file, kpis_file, metadata_file):
kpis = pd.read_csv(kpis_file)
if features_file:
features = pd.read_csv(features_file)
else:
features = 'default'
print(features)
metadata = parse_metadata(metadata_file)
exp_data = ExperimentData(metrics=kpis,
metadata=metadata,
features=features)
exp = Experiment(baseline_variant=metadata['baseline_variant'],
metrics_or_kpis=kpis,
metadata=metadata,
features=features)
return (exp.delta(), exp.sga())
def test_create_with_insufficient_data(self):
# should not work:
with self.assertRaises(KeyError):
ExperimentData(pd.DataFrame(columns=['entity', 'variant']),
metadata={
'experiment': 'test',
'source': 'none'
})
with self.assertRaises(KeyError):
ExperimentData(
pd.DataFrame(columns=['entity', 'variant', 'plums']),
metadata={
'experiment': 'test',
'source': 'none',
'primary_KPI': 'plums'
})
with self.assertRaises(ValueError):
ExperimentData(None)
# with self.assertRaises(KeyError):
# ExperimentData(pd.DataFrame())
with self.assertRaises(KeyError):
ExperimentData(pd.DataFrame(columns=['entity', 'variant']),
metadata=None)
# with self.assertRaises(KeyError):
# ExperimentData(
# pd.DataFrame(columns=['entity', 'treatment_start']),
# metadata={'experiment': 'fesf', 'source': 'random'},
# )
with self.assertRaises(KeyError):
ExperimentData(
pd.DataFrame(columns=['variant', 'treatment_start']),
metadata={
'experiment': 'fesf',
'source': 'random'
})
with self.assertRaises(KeyError):
ExperimentData(
pd.DataFrame(columns=['variant', 'entity', 'treatment_start']),
metadata={
'experiment': 'fesf',
'source': 'random',
'primary_KPI': 'something_not_there'
})
def test_init_with_aggregated_kpi(self):
"""Initialize ExperimentData with aggregated KPI data"""
D = ExperimentData(self.metrics, self.metadata, [4])
self.assertIsNone(D.kpis_time)
def test_initialize_with_metric_with_feature_df(self):
"""Initialize ExperimentData with metrics=DF, features=DF"""
metrics, meta = generate_random_data()
features = deepcopy(metrics)
D = ExperimentData(metrics, meta, features)
def test_initialize_with_metric_with_feature_list(self):
"""Initialize ExperimentData with metrics=DF, features=list"""
metrics, meta = generate_random_data()
D = ExperimentData(metrics, meta, [])
D = ExperimentData(metrics, meta, [4, 6])
def test_initialize_with_metric_with_feature_default(self):
"""Initialize ExperimentData with metrics=DF, features='default'"""
# metrics,meta = generate_random_data()
D = ExperimentData(self.metrics, self.metadata, 'default')
def test_outlier_filtering_treatment_exposure(self):
"""Check if scaling of the threshold works when the treatment_exposure is provided"""
self.metrics['treatment_exposure'] = 1000
D = ExperimentData(self.metrics[[
'entity', 'variant', 'normal_shifted', 'treatment_exposure'
]],
self.metadata,
features=[3])
D.filter_outliers(rules=[{
"metric": "normal_shifted",
"type": "threshold",
"value": -1.0,
"kind": "lower",
"time_interval": 1000
}])
n_filtered = D.metadata['n_filtered']
self.assertEqual(n_filtered, [1082])
#setting a larger long time_interval than the treatment_exposure does not affect the scaling of the threshold
D = ExperimentData(self.metrics[[
'entity', 'variant', 'normal_shifted', 'treatment_exposure'
]],
self.metadata,
features=[3])
D.filter_outliers(rules=[{
"metric": "normal_shifted",
"type": "threshold",
"value": -1.0,
"kind": "lower",
"time_interval": 10000
}])
self.assertEqual(D.metadata['n_filtered'], D.metadata['n_filtered'])
#setting a lower time_interval filters less, here so much less that filter will not be applied
D = ExperimentData(self.metrics[[
'entity', 'variant', 'normal_shifted', 'treatment_exposure'
]],
self.metadata,
features=[3])
D.filter_outliers(rules=[{
"metric": "normal_shifted",
"type": "threshold",
"value": -1.0,
"kind": "lower",
"time_interval": 100
}])
self.assertEqual(len(D.metadata['outlier_filter']), 0)
self.assertEqual(len(D.metadata['n_filtered']), 0)
def test_outlier_filtering(self):
"""Check outlier filtering functionality"""
#pick 1000 data points and make them outliers
metrics_outlier = self.metrics
idx = metrics_outlier.sample(1000).index
#make sure the values are below than -1 or above 1 then multiply
metrics_outlier.loc[idx, "normal_shifted_by_feature"] += np.sign(
metrics_outlier.loc[idx, "normal_shifted_by_feature"])
metrics_outlier.loc[idx, "normal_shifted_by_feature"] *= 10
# use 4 rules, one is not implemented and one does not apply, default settings
D = ExperimentData(metrics=metrics_outlier, metadata=self.metadata)
D.filter_outliers(rules=[
{
"metric": "normal_shifted_by_feature",
"type": "threshold",
"value": -10.0,
"kind": "lower"
},
{
"metric": "normal_shifted_by_feature",
"type": "threshold",
"value": 10.0,
"kind": "upper"
},
{
"metric": "normal_same",
"type": "threshold", #this does not apply
"value": 10.0,
"kind": "upper"
},
{
"metric": "normal_same",
"type": "water", #this is not implemented
"value": 10.0,
"kind": "both"
}
])
self.assertEqual(len(D.metadata['outlier_filter']),
2) # only are actually applied
self.assertEqual(len(D.metrics), 9000)
for i in idx:
self.assertEqual(D.metrics.ix[i].empty, True)
self.assertFalse('calc_thresh_value' in D.kpis.columns)
# use one rule, do not drop NaNs
D = ExperimentData(metrics=metrics_outlier, metadata=self.metadata)
temp_D = D
D.filter_outliers(rules=[{
"metric": "normal_shifted_by_feature",
"type": "threshold",
"value": 1.0,
"kind": "lower",
"time_interval": 30758400,
"treatment_stop_time": 30758500
}])
self.assertEqual(len(D.metadata['outlier_filter']), 1)
self.assertFalse('calc_thresh_value' in D.kpis.columns)
self.assertEqual(len(D.kpis), len(temp_D.kpis))
# use one rule, do not drop NaNs, do not drop threshold column
D = ExperimentData(metrics=metrics_outlier, metadata=self.metadata)
temp_D = D
D.filter_outliers(rules=[{
"metric": "normal_shifted_by_feature",
"type": "threshold",
"value": 1.0,
"kind": "lower",
"time_interval": 30758400,
"treatment_stop_time": 30758500
}],
drop_thresh=False)
self.assertEqual(len(D.metadata['outlier_filter']), 1)
self.assertTrue('calc_thresh_value' in D.kpis.columns)
self.assertEqual(len(D.kpis), len(temp_D.kpis))
