def _delta_all_variants(metric_df,
baseline_variant,
assume_normal=True,
percentiles=[2.5, 97.5],
min_observations=20,
nruns=10000,
relative=False,
weighted=False):
"""Applies delta to all variants, given a metric and a baseline variant.
metric_df has 4 columns: entity, variant, metric, reference_kpi
"""
baseline_metric = metric_df.iloc[:,
2][metric_df.iloc[:,
1] == baseline_variant]
baseline_weights = metric_df.iloc[:,
3][metric_df.iloc[:,
1] == baseline_variant]
if weighted:
# ASSUMPTIONS:
# - reference KPI is never NaN (such that sum works the same as np.nansum)
# - whenever the reference KPI is 0, it means the derived KPI is NaN,
# and therefore should not be counted (only works for ratio)
do_delta = (lambda f: delta_to_dataframe_all_variants(
f.columns[2],
*statx.delta(x=f.iloc[:, 2],
y=baseline_metric,
assume_normal=assume_normal,
percentiles=percentiles,
min_observations=min_observations,
nruns=nruns,
relative=relative,
x_weights=f.iloc[:, 3] / sum(f.iloc[:, 3]) * sum(
f.iloc[:, 3] != 0),
y_weights=baseline_weights / sum(baseline_weights) *
sum(baseline_weights != 0))))
else:
do_delta = (lambda f: delta_to_dataframe_all_variants(
f.columns[2],
*statx.delta(x=f.iloc[:, 2],
y=baseline_metric,
assume_normal=assume_normal,
percentiles=percentiles,
min_observations=min_observations,
nruns=nruns,
relative=relative)))
# Actual calculation
return metric_df.groupby('variant').apply(do_delta).unstack(0)
def _delta_all_variants(metric_df, baseline_variant, weighted=False,
deltaWorker=statx.make_delta()):
"""Applies delta to all variants, given a metric and a baseline variant.
metric_df has 4 columns: entity, variant, metric, reference_kpi
"""
baseline_metric = metric_df.iloc[:, 2][metric_df.iloc[:, 1] == baseline_variant]
baseline_weights = metric_df.iloc[:, 3][metric_df.iloc[:, 1] == baseline_variant]
if weighted:
# ASSUMPTIONS:
# - reference KPI is never NaN (such that sum works the same as np.nansum)
# - whenever the reference KPI is 0, it means the derived KPI is NaN,
# and therefore should not be counted (only works for ratio)
x_weights = lambda f: f.iloc[:,3]/sum(f.iloc[:,3])*sum(f.iloc[:,3]!=0)
y_weights = lambda f: baseline_weights/sum(baseline_weights)*sum(baseline_weights!=0)
else:
x_weights = lambda f: 1
y_weights = lambda f: 1
do_delta = (lambda f: delta_to_dataframe_all_variants(f.columns[2],
*deltaWorker(x=f.iloc[:,2],
y=baseline_metric,
x_weights = x_weights(f),
y_weights = y_weights(f))))
# Actual calculation
return metric_df.groupby('variant').apply(do_delta).unstack(0)
def do_delta(f, bin_name):
# find the corresponding bin in the baseline variant
baseline_metric = f.iloc[:, 2][(f.iloc[:, 0] == variants[1])]
out_df = pd.DataFrame()
for v in f['variant'].unique():
v_metric = f.iloc[:, 2][(f.iloc[:, 0] == v)]
df = delta_to_dataframe_all_variants(
f.columns[2],
*statx.delta(x=v_metric,
y=baseline_metric,
assume_normal=assume_normal,
percentiles=percentiles,
min_observations=min_observations,
nruns=nruns,
relative=relative))
# add new index levels for variant and binning
df['_tmp_bin_'] = bin_name
df['variant'] = v
df.set_index(['variant', '_tmp_bin_'], append=True, inplace=True)
df = df.reorder_levels([
'variant', '_tmp_bin_', 'metric', 'subgroup_metric',
'subgroup', 'statistic', 'pctile'
])
out_df = out_df.append(df)
return out_df
def _delta_all_variants(metric_df,
baseline_variant,
assume_normal=True,
percentiles=[2.5, 97.5],
min_observations=20,
nruns=10000,
relative=False,
weighted=False):
"""Applies delta to all variants, given a metric and a baseline variant.
metric_df has 4 columns: entity, variant, metric, reference_kpi
"""
baseline_metric = metric_df.iloc[:,
2][metric_df.iloc[:,
1] == baseline_variant]
baseline_weights = metric_df.iloc[:,
3][metric_df.iloc[:,
1] == baseline_variant]
if weighted:
do_delta = (lambda f: delta_to_dataframe_all_variants(
f.columns[2],
*statx.delta(x=f.iloc[:, 2],
y=baseline_metric,
assume_normal=assume_normal,
percentiles=percentiles,
min_observations=min_observations,
nruns=nruns,
relative=relative,
x_weights=f.iloc[:, 3] / sum(f.iloc[:, 3]) * len(
f.iloc[:, 3]),
y_weights=baseline_weights / sum(baseline_weights) *
len(baseline_weights))))
else:
do_delta = (lambda f: delta_to_dataframe_all_variants(
f.columns[2],
*statx.delta(x=f.iloc[:, 2],
y=baseline_metric,
assume_normal=assume_normal,
percentiles=percentiles,
min_observations=min_observations,
nruns=nruns,
relative=relative)))
# Actual calculation
return metric_df.groupby('variant').apply(do_delta).unstack(0)