def _simulate_experiment(self, config, static_vars, target_var, targets):
num_experiments = len(targets)
counter = collections.Counter()
self.mock_filewatcher.get_data.return_value = {config["name"]: config}
for target in targets:
experiment_vars = {target_var: target}
experiment_vars.update(static_vars)
user = experiment_vars.pop("user")
content = experiment_vars.pop("content")
experiments = self.get_experiment_client("test")
variant = experiments.variant(config["name"],
user_id=user["id"],
user_name=user["name"],
logged_in=user["logged_in"],
content_id=content["id"],
content_type=content["type"],
**experiment_vars)
if variant:
counter[variant] += 1
# this test will still probabilistically fail, but we can mitigate
# the likeliness of that happening
error_bar_percent = 100. / math.sqrt(num_experiments)
experiment = parse_experiment(config)
for variant, percent in iteritems(experiment.variants):
# Our actual percentage should be within our expected percent
# (expressed as a part of 100 rather than a fraction of 1)
# +- 1%.
measured_percent = (float(counter[variant]) /
num_experiments) * 100
self.assertAlmostEqual(measured_percent,
percent,
delta=error_bar_percent)
def test_choose_variant(self):
control_only = parse_experiment({
"id": 1,
"name": "control_only",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"control_1": 10,
"control_2": 10,
}
}
})
three_variants = parse_experiment({
"id": 1,
"name": "three_variants",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
'remove_vote_counters': 5,
'control_1': 10,
'control_2': 5,
}
}
})
three_variants_more = parse_experiment({
"id": 1,
"name": "three_variants_more",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
'remove_vote_counters': 15.6,
'control_1': 10,
'control_2': 20,
}
}
})
counters = collections.defaultdict(collections.Counter)
for bucket in range(control_only.num_buckets):
variant = control_only._choose_variant(bucket)
if variant:
counters[control_only.name][variant] += 1
# Ensure variant-choosing is deterministic.
self.assertEqual(variant, control_only._choose_variant(bucket))
variant = three_variants._choose_variant(bucket)
if variant:
counters[three_variants.name][variant] += 1
# Ensure variant-choosing is deterministic.
self.assertEqual(variant, three_variants._choose_variant(bucket))
previous_variant = variant
variant = three_variants_more._choose_variant(bucket)
if variant:
counters[three_variants_more.name][variant] += 1
# Ensure variant-choosing is deterministic.
self.assertEqual(variant,
three_variants_more._choose_variant(bucket))
# If previously we had a variant, we should still have the same one
# now.
if previous_variant:
self.assertEqual(variant, previous_variant)
for experiment in (control_only, three_variants, three_variants_more):
for variant, percentage in iteritems(experiment.variants):
count = counters[experiment.name][variant]
scaled_percentage = float(count) / (experiment.num_buckets /
100)
self.assertEqual(scaled_percentage, percentage)
# Test boundary conditions around the maximum percentage allowed for
# variants.
fifty_fifty = parse_experiment({
"id": 1,
"name": "fifty_fifty",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
'control_1': 50,
'control_2': 50,
}
}
})
almost_fifty_fifty = parse_experiment({
"id": 1,
"name": "almost_fifty_fifty",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
'control_1': 49,
'control_2': 51,
}
}
})
for bucket in range(fifty_fifty.num_buckets):
for experiment in (fifty_fifty, almost_fifty_fifty):
variant = experiment._choose_variant(bucket)
counters[experiment.name][variant] += 1
count = counters[fifty_fifty.name]['control_1']
scaled_percentage = float(count) / (fifty_fifty.num_buckets / 100)
self.assertEqual(scaled_percentage, 50)
count = counters[fifty_fifty.name]['control_2']
scaled_percentage = float(count) / (fifty_fifty.num_buckets / 100)
self.assertEqual(scaled_percentage, 50)
count = counters[almost_fifty_fifty.name]['control_1']
scaled_percentage = float(count) / (almost_fifty_fifty.num_buckets /
100)
self.assertEqual(scaled_percentage, 49)
count = counters[almost_fifty_fifty.name]['control_2']
scaled_percentage = float(count) / (almost_fifty_fifty.num_buckets /
100)
self.assertEqual(scaled_percentage, 50)