def test_change_shuffle_version_changes_bucketing(self):
cfg = get_simple_config()
experiment_version_1 = parse_experiment(cfg)
shuffle_cfg = get_simple_config()
shuffle_cfg["experiment"]["shuffle_version"] = 2
experiment_version_2 = parse_experiment(shuffle_cfg)
# Give ourselves enough users that we can get some reasonable amount of
# precision when checking amounts per bucket.
num_users = experiment_version_1.num_buckets * 100
fullnames = []
for i in range(num_users):
fullnames.append("t2_%s" % str(i))
counter = collections.Counter()
bucketing_changed = False
for fullname in fullnames:
bucket1 = experiment_version_1._calculate_bucket(fullname)
counter[bucket1] += 1
# Ensure bucketing is deterministic.
self.assertEqual(bucket1,
experiment_version_1._calculate_bucket(fullname))
bucket2 = experiment_version_2._calculate_bucket(fullname)
# check that the bucketing changed at some point. Can't compare
# bucket1 to bucket2 inline because sometimes the user will fall
# into both buckets, and test will fail. When a user doesn't match,
# break out of loop
if bucket1 != bucket2:
bucketing_changed = True
break
self.assertTrue(bucketing_changed)
def test_experiment_disabled(self):
experiments_cfg = {
"id": 1,
"name": "test_experiment",
"owner": "test",
"type": "single_variant",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"enabled": False,
"experiment": {
"variants": [
{
"name": "variant_1",
"size": 0.5
},
{
"name": "variant_2",
"size": 0.5
},
],
"experiment_version":
1,
},
}
experiment = parse_experiment(experiments_cfg)
variant = experiment.variant(user_id="t2_1")
self.assertIs(variant, None)
def test_subdomain_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {"subdomain": {"beta": "active", "www": "active"}},
"variants": {"active": 0},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, subdomain="beta"
),
"active",
)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, subdomain="BETA"
),
"active",
)
def test_url_enabled(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {"url_features": {"test_state": "active"}},
"variants": {"active": 0},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, url_features=["test_state"]
),
"active",
)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
url_features=["x", "test_state"],
),
"active",
)
def test_gold_disabled(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {"user_roles": {"gold": "active"}},
"variants": {"active": 0},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active"
)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, user_roles=[]
),
"active",
)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, user_roles=["admin"]
),
"active",
)
def test_targeting_in_config(self):
cfg = get_simple_config()
targeting_cfg = get_targeting_config()
cfg["experiment"]["targeting"] = targeting_cfg
experiment_with_targeting = parse_experiment(cfg)
self.assertTrue(
experiment_with_targeting.is_targeted(is_mod=True,
is_logged_in=True,
random_numeric=5))
def test_variant_returns_none_if_out_of_time_window(
self, choose_variant_mock):
choose_variant_mock.return_value = "fake_variant"
valid_cfg = get_simple_config()
experiment_valid = parse_experiment(valid_cfg)
expired_cfg = get_simple_config()
expired_cfg["stop_ts"] = time.time() - FIVE_DAYS
experiment_expired = parse_experiment(expired_cfg)
experiment_not_started_cfg = get_simple_config()
experiment_not_started_cfg["start_ts"] = time.time() + FIVE_DAYS
experiment_not_started = parse_experiment(experiment_not_started_cfg)
variant_valid = experiment_valid.variant(user_id="t2_1")
variant_expired = experiment_expired.variant(user_id="t2_1")
variant_not_started = experiment_not_started.variant(user_id="t2_1")
self.assertIsNot(variant_valid, None)
self.assertIs(variant_expired, None)
self.assertIs(variant_not_started, None)
def test_subreddit_not_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {"overrides": {"subreddit": {}}, "variants": {"active": 0}},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, subreddit="wtf"
),
"active",
)
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {"subreddit": {"wtfoobar": "active", "aww": "active"}},
"variants": {"active": 0},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, subreddit="wtf"
),
"active",
)
def simulate_percent_loggedout(wanted_percent):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"targeting": {"logged_in": [False]},
"variants": {"active": wanted_percent},
},
}
feature_flag = parse_experiment(cfg)
return (
feature_flag.variant(user_id="t2_%s" % str(i), logged_in=False) == "active"
for i in range(num_users)
)
def test_user_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"user_name": {"Gary": "active", "dave": "active", "ALL_UPPERCASE": "active"}
},
"variants": {"active": 0},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, user_name="Gary"
),
"active",
)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, user_name=self.user_name
),
"active",
)
all_uppercase_id = "t2_f00d"
all_uppercase_name = "ALL_UPPERCASE"
self.assertEqual(
feature_flag.variant(
user_id=all_uppercase_id, logged_in=True, user_name=all_uppercase_name
),
"active",
)
self.assertEqual(
feature_flag.variant(
user_id=all_uppercase_id, logged_in=True, user_name=all_uppercase_name.lower()
),
"active",
)
def test_calculate_bucket_value(self):
experiment = create_simple_experiment()
experiment.num_buckets = 1000
self.assertEqual(experiment._calculate_bucket("t2_1"), int(867))
seeded_cfg = {
"id": 1,
"name": "test_experiment",
"owner": "test",
"type": "single_variant",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"enabled": True,
"experiment": {
"variants": [
{
"name": "variant_1",
"size": 0.1
},
{
"name": "variant_2",
"size": 0.1
},
],
"experiment_version":
1,
"shuffle_version":
1,
"bucket_seed":
"some new seed",
},
}
seeded_experiment = parse_experiment(seeded_cfg)
self.assertNotEqual(seeded_experiment.seed, experiment.seed)
self.assertIsNot(seeded_experiment.seed, None)
seeded_experiment.num_buckets = 1000
self.assertEqual(seeded_experiment._calculate_bucket("t2_1"), int(924))
def test_bucket_val(self, choose_variant_mock):
choose_variant_mock.return_value = "fake_variant"
cfg = {
"id": 1,
"name": "test_experiment",
"owner": "test",
"type": "single_variant",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"enabled": True,
"experiment": {
"variants": [
{
"name": "variant_1",
"size": 0.5
},
{
"name": "variant_2",
"size": 0.5
},
],
"experiment_version":
1,
"bucket_val":
"new_bucket_val",
},
}
experiment = parse_experiment(cfg)
experiment._choose_variant = choose_variants_override
variant_default_bucket_val = experiment.variant(user_id="t2_1")
variant_new_bucket_val = experiment.variant(
new_bucket_val="some_value")
self.assertIs(variant_default_bucket_val, None)
self.assertIsNot(variant_new_bucket_val, None)
def test_get_override(self):
exp_config = get_simple_config()
override_config = get_simple_override_config()
exp_config["experiment"]["overrides"] = override_config
experiment_with_overrides = parse_experiment(exp_config)
self.assertEqual(
experiment_with_overrides.get_override(user_id="t2_1"),
"override_variant_1")
self.assertEqual(
experiment_with_overrides.get_override(user_id="t2_2"),
"override_variant_2")
self.assertEqual(
experiment_with_overrides.get_override(user_id="t2_3"),
"override_variant_3")
self.assertEqual(
experiment_with_overrides.get_override(user_id="t2_4"),
"override_variant_1")
def test_variant_call_with_overrides(self, choose_variant_mock):
choose_variant_mock.return_value = "mocked_variant"
exp_config = get_simple_config()
override_config = get_simple_override_config()
exp_config["experiment"]["overrides"] = override_config
experiment_with_overrides = parse_experiment(exp_config)
self.assertEqual(experiment_with_overrides.variant(user_id="t2_1"),
"override_variant_1")
self.assertEqual(experiment_with_overrides.variant(user_id="t2_2"),
"override_variant_2")
self.assertEqual(experiment_with_overrides.variant(user_id="t2_3"),
"override_variant_3")
self.assertEqual(experiment_with_overrides.variant(user_id="t2_4"),
"override_variant_1")
self.assertEqual(experiment_with_overrides.variant(user_id="t2_5"),
"mocked_variant")
def test_newer_than_only_on_logged_in_check(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"targeting": {"logged_in": [True], "user_name": ["gary"]},
"newer_than": int(time.time()) + THIRTY_DAYS,
"variants": {"active": 100},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, user_name="gary"
),
"active",
)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active"
)
def test_calculate_bucket_with_seed(self):
seeded_cfg = {
"id": 1,
"name": "test_experiment",
"owner": "test",
"type": "single_variant",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"enabled": True,
"experiment": {
"variants": [
{
"name": "variant_1",
"size": 0.1
},
{
"name": "variant_2",
"size": 0.1
},
],
"experiment_version":
1,
"shuffle_version":
1,
"bucket_seed":
"some_new_seed",
},
}
experiment = parse_experiment(seeded_cfg)
# Give ourselves enough users that we can get some reasonable amount of
# precision when checking amounts per bucket.
num_users = experiment.num_buckets * 2000
fullnames = []
for i in range(num_users):
fullnames.append("t2_%s" % str(i))
counter = collections.Counter()
bucketing_changed = False
for fullname in fullnames:
self.assertEqual(experiment.seed, "some_new_seed")
bucket1 = experiment._calculate_bucket(fullname)
counter[bucket1] += 1
# Ensure bucketing is deterministic.
self.assertEqual(bucket1, experiment._calculate_bucket(fullname))
current_seed = experiment.seed
experiment._seed = "newstring"
bucket2 = experiment._calculate_bucket(fullname)
experiment._seed = current_seed
# check that the bucketing changed at some point. Can't compare
# bucket1 to bucket2 inline because sometimes the user will fall
# into both buckets, and test will fail.
if bucket1 != bucket2:
bucketing_changed = True
self.assertTrue(bucketing_changed)
for bucket in range(experiment.num_buckets):
# We want an even distribution across buckets.
expected = num_users / experiment.num_buckets
actual = counter[bucket]
# Calculating the percentage difference instead of looking at the
# raw difference scales better as we change NUM_USERS.
percent_equal = float(actual) / expected
self.assertAlmostEqual(percent_equal,
1.0,
delta=0.10,
msg="bucket: %s" % bucket)
def test_multiple(self):
# is_admin, globally off should still be False
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"global_override": None,
"experiment": {
"overrides": {"user_roles": {"admin": "active"}},
"variants": {"active": 0},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, user_roles=["admin"]
),
"active",
)
# globally on but not admin should still be True
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"global_override": "active",
"experiment": {
"overrides": {"user_roles": {"admin": "active"}},
"variants": {"active": 0},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, user_roles=["admin"]
),
"active",
)
self.assertEqual(
feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active"
)
# no URL but admin should still be True
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"user_roles": {"admin": "active"},
"url_features": {"test_featurestate": "active"},
},
"variants": {"active": 0},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id, logged_in=self.user_logged_in, user_roles=["admin"]
),
"active",
)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
url_features=["test_featurestate"],
),
"active",
)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active"
)
def create_simple_experiment():
cfg = get_simple_config()
experiment = parse_experiment(cfg)
return experiment
