def test_change_shuffle_version_changes_bucketing(self): cfg = get_simple_config() experiment_version_1 = parse_experiment(cfg) shuffle_cfg = get_simple_config() shuffle_cfg["experiment"]["shuffle_version"] = 2 experiment_version_2 = parse_experiment(shuffle_cfg) # Give ourselves enough users that we can get some reasonable amount of # precision when checking amounts per bucket. num_users = experiment_version_1.num_buckets * 100 fullnames = [] for i in range(num_users): fullnames.append("t2_%s" % str(i)) counter = collections.Counter() bucketing_changed = False for fullname in fullnames: bucket1 = experiment_version_1._calculate_bucket(fullname) counter[bucket1] += 1 # Ensure bucketing is deterministic. self.assertEqual(bucket1, experiment_version_1._calculate_bucket(fullname)) bucket2 = experiment_version_2._calculate_bucket(fullname) # check that the bucketing changed at some point. Can't compare # bucket1 to bucket2 inline because sometimes the user will fall # into both buckets, and test will fail. When a user doesn't match, # break out of loop if bucket1 != bucket2: bucketing_changed = True break self.assertTrue(bucketing_changed)
def test_experiment_disabled(self): experiments_cfg = { "id": 1, "name": "test_experiment", "owner": "test", "type": "single_variant", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "enabled": False, "experiment": { "variants": [ { "name": "variant_1", "size": 0.5 }, { "name": "variant_2", "size": 0.5 }, ], "experiment_version": 1, }, } experiment = parse_experiment(experiments_cfg) variant = experiment.variant(user_id="t2_1") self.assertIs(variant, None)
def test_subdomain_in(self): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": {"subdomain": {"beta": "active", "www": "active"}}, "variants": {"active": 0}, }, } feature_flag = parse_experiment(cfg) self.assertEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, subdomain="beta" ), "active", ) self.assertEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, subdomain="BETA" ), "active", )
def test_url_enabled(self): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": {"url_features": {"test_state": "active"}}, "variants": {"active": 0}, }, } feature_flag = parse_experiment(cfg) self.assertEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, url_features=["test_state"] ), "active", ) self.assertEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, url_features=["x", "test_state"], ), "active", )
def test_gold_disabled(self): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": {"user_roles": {"gold": "active"}}, "variants": {"active": 0}, }, } feature_flag = parse_experiment(cfg) self.assertNotEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active" ) self.assertNotEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, user_roles=[] ), "active", ) self.assertNotEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, user_roles=["admin"] ), "active", )
def test_targeting_in_config(self): cfg = get_simple_config() targeting_cfg = get_targeting_config() cfg["experiment"]["targeting"] = targeting_cfg experiment_with_targeting = parse_experiment(cfg) self.assertTrue( experiment_with_targeting.is_targeted(is_mod=True, is_logged_in=True, random_numeric=5))
def test_variant_returns_none_if_out_of_time_window( self, choose_variant_mock): choose_variant_mock.return_value = "fake_variant" valid_cfg = get_simple_config() experiment_valid = parse_experiment(valid_cfg) expired_cfg = get_simple_config() expired_cfg["stop_ts"] = time.time() - FIVE_DAYS experiment_expired = parse_experiment(expired_cfg) experiment_not_started_cfg = get_simple_config() experiment_not_started_cfg["start_ts"] = time.time() + FIVE_DAYS experiment_not_started = parse_experiment(experiment_not_started_cfg) variant_valid = experiment_valid.variant(user_id="t2_1") variant_expired = experiment_expired.variant(user_id="t2_1") variant_not_started = experiment_not_started.variant(user_id="t2_1") self.assertIsNot(variant_valid, None) self.assertIs(variant_expired, None) self.assertIs(variant_not_started, None)
def test_subreddit_not_in(self): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": {"overrides": {"subreddit": {}}, "variants": {"active": 0}}, } feature_flag = parse_experiment(cfg) self.assertNotEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, subreddit="wtf" ), "active", ) cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": {"subreddit": {"wtfoobar": "active", "aww": "active"}}, "variants": {"active": 0}, }, } feature_flag = parse_experiment(cfg) self.assertNotEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, subreddit="wtf" ), "active", )
def simulate_percent_loggedout(wanted_percent): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "targeting": {"logged_in": [False]}, "variants": {"active": wanted_percent}, }, } feature_flag = parse_experiment(cfg) return ( feature_flag.variant(user_id="t2_%s" % str(i), logged_in=False) == "active" for i in range(num_users) )
def test_user_in(self): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": { "user_name": {"Gary": "active", "dave": "active", "ALL_UPPERCASE": "active"} }, "variants": {"active": 0}, }, } feature_flag = parse_experiment(cfg) self.assertEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, user_name="Gary" ), "active", ) self.assertEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, user_name=self.user_name ), "active", ) all_uppercase_id = "t2_f00d" all_uppercase_name = "ALL_UPPERCASE" self.assertEqual( feature_flag.variant( user_id=all_uppercase_id, logged_in=True, user_name=all_uppercase_name ), "active", ) self.assertEqual( feature_flag.variant( user_id=all_uppercase_id, logged_in=True, user_name=all_uppercase_name.lower() ), "active", )
def test_calculate_bucket_value(self): experiment = create_simple_experiment() experiment.num_buckets = 1000 self.assertEqual(experiment._calculate_bucket("t2_1"), int(867)) seeded_cfg = { "id": 1, "name": "test_experiment", "owner": "test", "type": "single_variant", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "enabled": True, "experiment": { "variants": [ { "name": "variant_1", "size": 0.1 }, { "name": "variant_2", "size": 0.1 }, ], "experiment_version": 1, "shuffle_version": 1, "bucket_seed": "some new seed", }, } seeded_experiment = parse_experiment(seeded_cfg) self.assertNotEqual(seeded_experiment.seed, experiment.seed) self.assertIsNot(seeded_experiment.seed, None) seeded_experiment.num_buckets = 1000 self.assertEqual(seeded_experiment._calculate_bucket("t2_1"), int(924))
def test_bucket_val(self, choose_variant_mock): choose_variant_mock.return_value = "fake_variant" cfg = { "id": 1, "name": "test_experiment", "owner": "test", "type": "single_variant", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "enabled": True, "experiment": { "variants": [ { "name": "variant_1", "size": 0.5 }, { "name": "variant_2", "size": 0.5 }, ], "experiment_version": 1, "bucket_val": "new_bucket_val", }, } experiment = parse_experiment(cfg) experiment._choose_variant = choose_variants_override variant_default_bucket_val = experiment.variant(user_id="t2_1") variant_new_bucket_val = experiment.variant( new_bucket_val="some_value") self.assertIs(variant_default_bucket_val, None) self.assertIsNot(variant_new_bucket_val, None)
def test_get_override(self): exp_config = get_simple_config() override_config = get_simple_override_config() exp_config["experiment"]["overrides"] = override_config experiment_with_overrides = parse_experiment(exp_config) self.assertEqual( experiment_with_overrides.get_override(user_id="t2_1"), "override_variant_1") self.assertEqual( experiment_with_overrides.get_override(user_id="t2_2"), "override_variant_2") self.assertEqual( experiment_with_overrides.get_override(user_id="t2_3"), "override_variant_3") self.assertEqual( experiment_with_overrides.get_override(user_id="t2_4"), "override_variant_1")
def test_variant_call_with_overrides(self, choose_variant_mock): choose_variant_mock.return_value = "mocked_variant" exp_config = get_simple_config() override_config = get_simple_override_config() exp_config["experiment"]["overrides"] = override_config experiment_with_overrides = parse_experiment(exp_config) self.assertEqual(experiment_with_overrides.variant(user_id="t2_1"), "override_variant_1") self.assertEqual(experiment_with_overrides.variant(user_id="t2_2"), "override_variant_2") self.assertEqual(experiment_with_overrides.variant(user_id="t2_3"), "override_variant_3") self.assertEqual(experiment_with_overrides.variant(user_id="t2_4"), "override_variant_1") self.assertEqual(experiment_with_overrides.variant(user_id="t2_5"), "mocked_variant")
def test_newer_than_only_on_logged_in_check(self): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "targeting": {"logged_in": [True], "user_name": ["gary"]}, "newer_than": int(time.time()) + THIRTY_DAYS, "variants": {"active": 100}, }, } feature_flag = parse_experiment(cfg) self.assertEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, user_name="gary" ), "active", ) self.assertNotEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active" )
def test_calculate_bucket_with_seed(self): seeded_cfg = { "id": 1, "name": "test_experiment", "owner": "test", "type": "single_variant", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "enabled": True, "experiment": { "variants": [ { "name": "variant_1", "size": 0.1 }, { "name": "variant_2", "size": 0.1 }, ], "experiment_version": 1, "shuffle_version": 1, "bucket_seed": "some_new_seed", }, } experiment = parse_experiment(seeded_cfg) # Give ourselves enough users that we can get some reasonable amount of # precision when checking amounts per bucket. num_users = experiment.num_buckets * 2000 fullnames = [] for i in range(num_users): fullnames.append("t2_%s" % str(i)) counter = collections.Counter() bucketing_changed = False for fullname in fullnames: self.assertEqual(experiment.seed, "some_new_seed") bucket1 = experiment._calculate_bucket(fullname) counter[bucket1] += 1 # Ensure bucketing is deterministic. self.assertEqual(bucket1, experiment._calculate_bucket(fullname)) current_seed = experiment.seed experiment._seed = "newstring" bucket2 = experiment._calculate_bucket(fullname) experiment._seed = current_seed # check that the bucketing changed at some point. Can't compare # bucket1 to bucket2 inline because sometimes the user will fall # into both buckets, and test will fail. if bucket1 != bucket2: bucketing_changed = True self.assertTrue(bucketing_changed) for bucket in range(experiment.num_buckets): # We want an even distribution across buckets. expected = num_users / experiment.num_buckets actual = counter[bucket] # Calculating the percentage difference instead of looking at the # raw difference scales better as we change NUM_USERS. percent_equal = float(actual) / expected self.assertAlmostEqual(percent_equal, 1.0, delta=0.10, msg="bucket: %s" % bucket)
def test_multiple(self): # is_admin, globally off should still be False cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "global_override": None, "experiment": { "overrides": {"user_roles": {"admin": "active"}}, "variants": {"active": 0}, }, } feature_flag = parse_experiment(cfg) self.assertNotEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, user_roles=["admin"] ), "active", ) # globally on but not admin should still be True cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "global_override": "active", "experiment": { "overrides": {"user_roles": {"admin": "active"}}, "variants": {"active": 0}, }, } feature_flag = parse_experiment(cfg) self.assertEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, user_roles=["admin"] ), "active", ) self.assertEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active" ) # no URL but admin should still be True cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": { "user_roles": {"admin": "active"}, "url_features": {"test_featurestate": "active"}, }, "variants": {"active": 0}, }, } feature_flag = parse_experiment(cfg) self.assertEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, user_roles=["admin"] ), "active", ) self.assertEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, url_features=["test_featurestate"], ), "active", ) self.assertNotEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active" )
def create_simple_experiment(): cfg = get_simple_config() experiment = parse_experiment(cfg) return experiment