def test_change_shuffle_version_changes_bucketing(self):
cfg = get_simple_config()
experiment_version_1 = parse_experiment(cfg)
shuffle_cfg = get_simple_config()
shuffle_cfg["experiment"]["shuffle_version"] = 2
experiment_version_2 = parse_experiment(shuffle_cfg)
# Give ourselves enough users that we can get some reasonable amount of
# precision when checking amounts per bucket.
num_users = experiment_version_1.num_buckets * 100
fullnames = []
for i in range(num_users):
fullnames.append("t2_%s" % str(i))
counter = collections.Counter()
bucketing_changed = False
for fullname in fullnames:
bucket1 = experiment_version_1._calculate_bucket(fullname)
counter[bucket1] += 1
# Ensure bucketing is deterministic.
self.assertEqual(bucket1,
experiment_version_1._calculate_bucket(fullname))
bucket2 = experiment_version_2._calculate_bucket(fullname)
# check that the bucketing changed at some point. Can't compare
# bucket1 to bucket2 inline because sometimes the user will fall
# into both buckets, and test will fail. When a user doesn't match,
# break out of loop
if bucket1 != bucket2:
bucketing_changed = True
break
self.assertTrue(bucketing_changed)
def test_subdomain_not_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"subdomain": {}
},
"variants": {
"active": 0
}
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
subdomain="beta"),
"active",
)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
subdomain=""),
"active",
)
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"subdomain": {
"www": "active",
"betanauts": "active"
}
},
"variants": {
"active": 0
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
subdomain="beta"),
"active",
)
def test_url_disabled(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"url_features": {
"test_state": "active"
}
},
"variants": {
"active": 0
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in), "active")
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
url_features=["x"]),
"active",
)
def test_subreddit_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"subreddit": {
"WTF": "active",
"aww": "active"
}
},
"variants": {
"active": 0
},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
subreddit="WTF"),
"active",
)
self.assertEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
subreddit="wtf"),
"active",
)
def test_return_override_variant_without_bucket_val(self):
experiment = parse_experiment({
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"user_name": {
"gary": "active"
}
},
"variants": {
"active": 10,
"control_1": 10,
"control_2": 20
},
},
})
variant = experiment.variant(user_name="gary")
self.assertEqual(variant, "active")
variant = experiment.variant()
self.assertEqual(variant, None)
def _simulate_experiment(self, config, static_vars, target_var, targets):
num_experiments = len(targets)
counter = collections.Counter()
self.mock_filewatcher.get_data.return_value = {config["name"]: config}
for target in targets:
experiment_vars = {target_var: target}
experiment_vars.update(static_vars)
user = experiment_vars.pop("user")
content = experiment_vars.pop("content")
experiments = self.get_experiment_client("test")
variant = experiments.variant(
config["name"],
user_id=user["id"],
user_name=user["name"],
logged_in=user["logged_in"],
content_id=content["id"],
content_type=content["type"],
**experiment_vars,
)
if variant:
counter[variant] += 1
# this test will still probabilistically fail, but we can mitigate
# the likeliness of that happening
error_bar_percent = 100.0 / math.sqrt(num_experiments)
experiment = parse_experiment(config)
for variant, percent in experiment.variants.items():
# Our actual percentage should be within our expected percent
# (expressed as a part of 100 rather than a fraction of 1)
# +- 1%.
measured_percent = (float(counter[variant]) /
num_experiments) * 100
self.assertAlmostEqual(measured_percent,
percent,
delta=error_bar_percent)
def test_experiment_disabled(self):
experiments_cfg = {
"id": 1,
"name": "test_experiment",
"owner": "test",
"type": "single_variant",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"enabled": False,
"experiment": {
"variants": [
{
"name": "variant_1",
"size": 0.5
},
{
"name": "variant_2",
"size": 0.5
},
],
"experiment_version":
1,
},
}
experiment = parse_experiment(experiments_cfg)
variant = experiment.variant(user_id="t2_1")
self.assertIs(variant, None)
def test_newer_than_only_on_logged_in_check(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"targeting": {
"logged_in": [True],
"user_name": ["gary"]
},
"newer_than": int(time.time()) + THIRTY_DAYS,
"variants": {
"active": 100
},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
user_name="gary"),
"active",
)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in), "active")
def test_calculate_bucket_with_seed(self):
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"control_1": 10,
"control_2": 10
},
"seed": "itscoldintheoffice",
},
}
experiment = parse_experiment(cfg)
# Give ourselves enough users that we can get some reasonable amount of
# precision when checking amounts per bucket.
num_users = experiment.num_buckets * 1000
fullnames = []
for i in range(num_users):
fullnames.append("t2_%s" % str(i))
counter = collections.Counter()
bucketing_changed = False
for fullname in fullnames:
self.assertEqual(experiment.seed, "itscoldintheoffice")
bucket1 = experiment._calculate_bucket(fullname)
counter[bucket1] += 1
# Ensure bucketing is deterministic.
self.assertEqual(bucket1, experiment._calculate_bucket(fullname))
current_seed = experiment.seed
experiment.seed = "newstring"
bucket2 = experiment._calculate_bucket(fullname)
experiment.seed = current_seed
# check that the bucketing changed at some point. Can't compare
# bucket1 to bucket2 inline because sometimes the user will fall
# into both buckets, and test will fail
if bucket1 != bucket2:
bucketing_changed = True
self.assertTrue(bucketing_changed)
for bucket in range(experiment.num_buckets):
# We want an even distribution across buckets.
expected = num_users / experiment.num_buckets
actual = counter[bucket]
# Calculating the percentage difference instead of looking at the
# raw difference scales better as we change NUM_USERS.
percent_equal = float(actual) / expected
self.assertAlmostEqual(percent_equal,
1.0,
delta=0.10,
msg="bucket: %s" % bucket)
def test_targeting_in_config(self):
cfg = get_simple_config()
targeting_cfg = get_targeting_config()
cfg["experiment"]["targeting"] = targeting_cfg
experiment_with_targeting = parse_experiment(cfg)
self.assertTrue(
experiment_with_targeting.is_targeted(is_mod=True,
is_logged_in=True,
random_numeric=5))
def test_variant_returns_none_if_out_of_time_window(
self, choose_variant_mock):
choose_variant_mock.return_value = "fake_variant"
valid_cfg = get_simple_config()
experiment_valid = parse_experiment(valid_cfg)
expired_cfg = get_simple_config()
expired_cfg["stop_ts"] = time.time() - FIVE_DAYS
experiment_expired = parse_experiment(expired_cfg)
experiment_not_started_cfg = get_simple_config()
experiment_not_started_cfg["start_ts"] = time.time() + FIVE_DAYS
experiment_not_started = parse_experiment(experiment_not_started_cfg)
variant_valid = experiment_valid.variant(user_id="t2_1")
variant_expired = experiment_expired.variant(user_id="t2_1")
variant_not_started = experiment_not_started.variant(user_id="t2_1")
self.assertIsNot(variant_valid, None)
self.assertIs(variant_expired, None)
self.assertIs(variant_not_started, None)
def test_calculate_bucket_value(self):
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"control_1": 10,
"control_2": 10
}
},
}
experiment = parse_experiment(cfg)
experiment.num_buckets = 1000
self.assertEqual(experiment._calculate_bucket("t2_1"), int(236))
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"seed": "test-seed",
"variants": {
"control_1": 10,
"control_2": 10
}
},
}
seeded_experiment = parse_experiment(cfg)
self.assertNotEqual(seeded_experiment.seed, experiment.seed)
self.assertIsNot(seeded_experiment.seed, None)
seeded_experiment.num_buckets = 1000
self.assertEqual(seeded_experiment._calculate_bucket("t2_1"), int(595))
def _get_experiment(self, name: str) -> Optional[Experiment]:
if name not in self._experiment_cache:
experiment_config = self._get_config(name)
if not experiment_config:
experiment = None
else:
try:
experiment = parse_experiment(experiment_config)
except Exception as err:
logger.error("Invalid configuration for experiment %s: %s",
name, err)
return None
self._experiment_cache[name] = experiment
return self._experiment_cache[name]
def test_user_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"user_name": {
"Gary": "active",
"dave": "active",
"ALL_UPPERCASE": "active"
}
},
"variants": {
"active": 0
},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
user_name="Gary"),
"active",
)
self.assertEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
user_name=self.user_name),
"active",
)
all_uppercase_id = "t2_f00d"
all_uppercase_name = "ALL_UPPERCASE"
self.assertEqual(
feature_flag.variant(user_id=all_uppercase_id,
logged_in=True,
user_name=all_uppercase_name),
"active",
)
self.assertEqual(
feature_flag.variant(user_id=all_uppercase_id,
logged_in=True,
user_name=all_uppercase_name.lower()),
"active",
)
def test_no_version_allowed(self):
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"control_1": 10,
"control_2": 10
}
},
}
experiment = parse_experiment(cfg)
self.assertTrue(isinstance(experiment, R2Experiment))
self.assertTrue(experiment.should_log_bucketing())
self.assertIs(experiment.version, None)
def test_calculate_bucket(self):
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"control_1": 10,
"control_2": 10
}
},
}
experiment = parse_experiment(cfg)
# Give ourselves enough users that we can get some reasonable amount of
# precision when checking amounts per bucket.
num_users = experiment.num_buckets * 1000
fullnames = []
for i in range(num_users):
fullnames.append("t2_%s" % str(i))
counter = collections.Counter()
for fullname in fullnames:
bucket = experiment._calculate_bucket(fullname)
counter[bucket] += 1
# Ensure bucketing is deterministic.
self.assertEqual(bucket, experiment._calculate_bucket(fullname))
for bucket in range(experiment.num_buckets):
# We want an even distribution across buckets.
expected = num_users / experiment.num_buckets
actual = counter[bucket]
# Calculating the percentage difference instead of looking at the
# raw difference scales better as we change num_users.
percent_equal = float(actual) / expected
self.assertAlmostEqual(percent_equal,
1.0,
delta=0.10,
msg="bucket: %s" % bucket)
def test_r2_type_returns_r2_experiment(self):
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"control_1": 10,
"control_2": 10
}
},
}
experiment = parse_experiment(cfg)
self.assertTrue(isinstance(experiment, R2Experiment))
self.assertTrue(experiment.should_log_bucketing())
self.assertEqual(experiment.version, "1")
def _get_experiment(self, name: str) -> Optional[Experiment]:
if name in self._global_cache:
return self._global_cache[name]
if self._cfg_data is None:
warn_deprecated("config_watcher will be removed in Baseplate 2.0.")
self._cfg_data = self._get_config()
if name not in self._cfg_data:
logger.info("Experiment <%r> not found in experiment config", name)
return None
try:
experiment = parse_experiment(self._cfg_data[name])
self._global_cache[name] = experiment
return experiment
except Exception as err:
logger.error("Invalid configuration for experiment %s: %s", name,
err)
return None
def test_after_expires_returns_forced_variant(self):
expires = (datetime.now() - timedelta(days=30)).strftime(ISO_DATE_FMT)
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"expires": expires,
"enabled": True,
"experiment": {
"id": 1,
"name": "test",
"variants": {
"control_1": 10,
"control_2": 10
}
},
}
experiment = parse_experiment(cfg)
self.assertTrue(isinstance(experiment, ForcedVariantExperiment))
def simulate_percent_loggedout(wanted_percent):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"targeting": {
"logged_in": [False]
},
"variants": {
"active": wanted_percent
},
},
}
feature_flag = parse_experiment(cfg)
return (feature_flag.variant(user_id="t2_%s" % str(i),
logged_in=False) == "active"
for i in range(num_users))
def test_before_start_ts_returns_forced_variant(self):
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() + THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS * 2,
"enabled": True,
"experiment": {
"id": 1,
"name": "test",
"variants": {
"control_1": 10,
"control_2": 10
}
},
}
experiment = parse_experiment(cfg)
self.assertTrue(isinstance(experiment, ForcedVariantExperiment))
def test_calculate_bucket_value(self):
experiment = create_simple_experiment()
experiment.num_buckets = 1000
self.assertEqual(experiment._calculate_bucket("t2_1"), int(867))
seeded_cfg = {
"id": 1,
"name": "test_experiment",
"owner": "test",
"type": "single_variant",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"enabled": True,
"experiment": {
"variants": [
{
"name": "variant_1",
"size": 0.1
},
{
"name": "variant_2",
"size": 0.1
},
],
"experiment_version":
1,
"shuffle_version":
1,
"bucket_seed":
"some new seed",
},
}
seeded_experiment = parse_experiment(seeded_cfg)
self.assertNotEqual(seeded_experiment.seed, experiment.seed)
self.assertIsNot(seeded_experiment.seed, None)
seeded_experiment.num_buckets = 1000
self.assertEqual(seeded_experiment._calculate_bucket("t2_1"), int(924))
def test_unknown_type_returns_null_experiment(self):
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "unknown",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"id": 1,
"name": "test",
"variants": {
"control_1": 10,
"control_2": 10
}
},
}
experiment = parse_experiment(cfg)
self.assertTrue(isinstance(experiment, ForcedVariantExperiment))
self.assertIs(experiment.variant(), None)
self.assertFalse(experiment.should_log_bucketing())
def test_bucket_val(self, choose_variant_mock):
choose_variant_mock.return_value = "fake_variant"
cfg = {
"id": 1,
"name": "test_experiment",
"owner": "test",
"type": "single_variant",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"enabled": True,
"experiment": {
"variants": [
{
"name": "variant_1",
"size": 0.5
},
{
"name": "variant_2",
"size": 0.5
},
],
"experiment_version":
1,
"bucket_val":
"new_bucket_val",
},
}
experiment = parse_experiment(cfg)
experiment._choose_variant = choose_variants_override
variant_default_bucket_val = experiment.variant(user_id="t2_1")
variant_new_bucket_val = experiment.variant(
new_bucket_val="some_value")
self.assertIs(variant_default_bucket_val, None)
self.assertIsNot(variant_new_bucket_val, None)
def test_get_override(self):
exp_config = get_simple_config()
override_config = get_simple_override_config()
exp_config["experiment"]["overrides"] = override_config
experiment_with_overrides = parse_experiment(exp_config)
self.assertEqual(
experiment_with_overrides.get_override(user_id="t2_1"),
"override_variant_1")
self.assertEqual(
experiment_with_overrides.get_override(user_id="t2_2"),
"override_variant_2")
self.assertEqual(
experiment_with_overrides.get_override(user_id="t2_3"),
"override_variant_3")
self.assertEqual(
experiment_with_overrides.get_override(user_id="t2_4"),
"override_variant_1")
def test_expires_ignores_start_ts(self):
expires = (datetime.now() + timedelta(days=30)).strftime(ISO_DATE_FMT)
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() + THIRTY_DAYS,
"expires": expires,
"enabled": True,
"experiment": {
"id": 1,
"name": "test",
"variants": {
"control_1": 10,
"control_2": 10
}
},
}
experiment = parse_experiment(cfg)
self.assertFalse(isinstance(experiment, ForcedVariantExperiment))
def test_variant_call_with_overrides(self, choose_variant_mock):
choose_variant_mock.return_value = "mocked_variant"
exp_config = get_simple_config()
override_config = get_simple_override_config()
exp_config["experiment"]["overrides"] = override_config
experiment_with_overrides = parse_experiment(exp_config)
self.assertEqual(experiment_with_overrides.variant(user_id="t2_1"),
"override_variant_1")
self.assertEqual(experiment_with_overrides.variant(user_id="t2_2"),
"override_variant_2")
self.assertEqual(experiment_with_overrides.variant(user_id="t2_3"),
"override_variant_3")
self.assertEqual(experiment_with_overrides.variant(user_id="t2_4"),
"override_variant_1")
self.assertEqual(experiment_with_overrides.variant(user_id="t2_5"),
"mocked_variant")
def test_non_string_override_value(self):
experiment = parse_experiment({
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"logged_in": {
True: "active"
}
},
"variants": {
"active": 10,
"control_1": 10,
"control_2": 20
},
},
})
variant = experiment.variant(logged_in=True)
self.assertEqual(variant, "active")
def test_multiple(self):
# is_admin, globally off should still be False
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"global_override": None,
"experiment": {
"overrides": {
"user_roles": {
"admin": "active"
}
},
"variants": {
"active": 0
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
user_roles=["admin"]),
"active",
)
# globally on but not admin should still be True
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"global_override": "active",
"experiment": {
"overrides": {
"user_roles": {
"admin": "active"
}
},
"variants": {
"active": 0
},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
user_roles=["admin"]),
"active",
)
self.assertEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in), "active")
# no URL but admin should still be True
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"user_roles": {
"admin": "active"
},
"url_features": {
"test_featurestate": "active"
},
},
"variants": {
"active": 0
},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
user_roles=["admin"]),
"active",
)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
url_features=["test_featurestate"],
),
"active",
)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in), "active")
def create_simple_experiment():
cfg = get_simple_config()
experiment = parse_experiment(cfg)
return experiment
