def test_change_shuffle_version_changes_bucketing(self): cfg = get_simple_config() experiment_version_1 = parse_experiment(cfg) shuffle_cfg = get_simple_config() shuffle_cfg["experiment"]["shuffle_version"] = 2 experiment_version_2 = parse_experiment(shuffle_cfg) # Give ourselves enough users that we can get some reasonable amount of # precision when checking amounts per bucket. num_users = experiment_version_1.num_buckets * 100 fullnames = [] for i in range(num_users): fullnames.append("t2_%s" % str(i)) counter = collections.Counter() bucketing_changed = False for fullname in fullnames: bucket1 = experiment_version_1._calculate_bucket(fullname) counter[bucket1] += 1 # Ensure bucketing is deterministic. self.assertEqual(bucket1, experiment_version_1._calculate_bucket(fullname)) bucket2 = experiment_version_2._calculate_bucket(fullname) # check that the bucketing changed at some point. Can't compare # bucket1 to bucket2 inline because sometimes the user will fall # into both buckets, and test will fail. When a user doesn't match, # break out of loop if bucket1 != bucket2: bucketing_changed = True break self.assertTrue(bucketing_changed)
def test_subdomain_not_in(self): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": { "subdomain": {} }, "variants": { "active": 0 } }, } feature_flag = parse_experiment(cfg) self.assertNotEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, subdomain="beta"), "active", ) self.assertNotEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, subdomain=""), "active", ) cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": { "subdomain": { "www": "active", "betanauts": "active" } }, "variants": { "active": 0 }, }, } feature_flag = parse_experiment(cfg) self.assertNotEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, subdomain="beta"), "active", )
def test_url_disabled(self): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": { "url_features": { "test_state": "active" } }, "variants": { "active": 0 }, }, } feature_flag = parse_experiment(cfg) self.assertNotEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active") self.assertNotEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, url_features=["x"]), "active", )
def test_subreddit_in(self): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": { "subreddit": { "WTF": "active", "aww": "active" } }, "variants": { "active": 0 }, }, } feature_flag = parse_experiment(cfg) self.assertEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, subreddit="WTF"), "active", ) self.assertEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, subreddit="wtf"), "active", )
def test_return_override_variant_without_bucket_val(self): experiment = parse_experiment({ "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": { "user_name": { "gary": "active" } }, "variants": { "active": 10, "control_1": 10, "control_2": 20 }, }, }) variant = experiment.variant(user_name="gary") self.assertEqual(variant, "active") variant = experiment.variant() self.assertEqual(variant, None)
def _simulate_experiment(self, config, static_vars, target_var, targets): num_experiments = len(targets) counter = collections.Counter() self.mock_filewatcher.get_data.return_value = {config["name"]: config} for target in targets: experiment_vars = {target_var: target} experiment_vars.update(static_vars) user = experiment_vars.pop("user") content = experiment_vars.pop("content") experiments = self.get_experiment_client("test") variant = experiments.variant( config["name"], user_id=user["id"], user_name=user["name"], logged_in=user["logged_in"], content_id=content["id"], content_type=content["type"], **experiment_vars, ) if variant: counter[variant] += 1 # this test will still probabilistically fail, but we can mitigate # the likeliness of that happening error_bar_percent = 100.0 / math.sqrt(num_experiments) experiment = parse_experiment(config) for variant, percent in experiment.variants.items(): # Our actual percentage should be within our expected percent # (expressed as a part of 100 rather than a fraction of 1) # +- 1%. measured_percent = (float(counter[variant]) / num_experiments) * 100 self.assertAlmostEqual(measured_percent, percent, delta=error_bar_percent)
def test_experiment_disabled(self): experiments_cfg = { "id": 1, "name": "test_experiment", "owner": "test", "type": "single_variant", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "enabled": False, "experiment": { "variants": [ { "name": "variant_1", "size": 0.5 }, { "name": "variant_2", "size": 0.5 }, ], "experiment_version": 1, }, } experiment = parse_experiment(experiments_cfg) variant = experiment.variant(user_id="t2_1") self.assertIs(variant, None)
def test_newer_than_only_on_logged_in_check(self): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "targeting": { "logged_in": [True], "user_name": ["gary"] }, "newer_than": int(time.time()) + THIRTY_DAYS, "variants": { "active": 100 }, }, } feature_flag = parse_experiment(cfg) self.assertEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, user_name="gary"), "active", ) self.assertNotEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active")
def test_calculate_bucket_with_seed(self): cfg = { "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { "control_1": 10, "control_2": 10 }, "seed": "itscoldintheoffice", }, } experiment = parse_experiment(cfg) # Give ourselves enough users that we can get some reasonable amount of # precision when checking amounts per bucket. num_users = experiment.num_buckets * 1000 fullnames = [] for i in range(num_users): fullnames.append("t2_%s" % str(i)) counter = collections.Counter() bucketing_changed = False for fullname in fullnames: self.assertEqual(experiment.seed, "itscoldintheoffice") bucket1 = experiment._calculate_bucket(fullname) counter[bucket1] += 1 # Ensure bucketing is deterministic. self.assertEqual(bucket1, experiment._calculate_bucket(fullname)) current_seed = experiment.seed experiment.seed = "newstring" bucket2 = experiment._calculate_bucket(fullname) experiment.seed = current_seed # check that the bucketing changed at some point. Can't compare # bucket1 to bucket2 inline because sometimes the user will fall # into both buckets, and test will fail if bucket1 != bucket2: bucketing_changed = True self.assertTrue(bucketing_changed) for bucket in range(experiment.num_buckets): # We want an even distribution across buckets. expected = num_users / experiment.num_buckets actual = counter[bucket] # Calculating the percentage difference instead of looking at the # raw difference scales better as we change NUM_USERS. percent_equal = float(actual) / expected self.assertAlmostEqual(percent_equal, 1.0, delta=0.10, msg="bucket: %s" % bucket)
def test_targeting_in_config(self): cfg = get_simple_config() targeting_cfg = get_targeting_config() cfg["experiment"]["targeting"] = targeting_cfg experiment_with_targeting = parse_experiment(cfg) self.assertTrue( experiment_with_targeting.is_targeted(is_mod=True, is_logged_in=True, random_numeric=5))
def test_variant_returns_none_if_out_of_time_window( self, choose_variant_mock): choose_variant_mock.return_value = "fake_variant" valid_cfg = get_simple_config() experiment_valid = parse_experiment(valid_cfg) expired_cfg = get_simple_config() expired_cfg["stop_ts"] = time.time() - FIVE_DAYS experiment_expired = parse_experiment(expired_cfg) experiment_not_started_cfg = get_simple_config() experiment_not_started_cfg["start_ts"] = time.time() + FIVE_DAYS experiment_not_started = parse_experiment(experiment_not_started_cfg) variant_valid = experiment_valid.variant(user_id="t2_1") variant_expired = experiment_expired.variant(user_id="t2_1") variant_not_started = experiment_not_started.variant(user_id="t2_1") self.assertIsNot(variant_valid, None) self.assertIs(variant_expired, None) self.assertIs(variant_not_started, None)
def test_calculate_bucket_value(self): cfg = { "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { "control_1": 10, "control_2": 10 } }, } experiment = parse_experiment(cfg) experiment.num_buckets = 1000 self.assertEqual(experiment._calculate_bucket("t2_1"), int(236)) cfg = { "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "seed": "test-seed", "variants": { "control_1": 10, "control_2": 10 } }, } seeded_experiment = parse_experiment(cfg) self.assertNotEqual(seeded_experiment.seed, experiment.seed) self.assertIsNot(seeded_experiment.seed, None) seeded_experiment.num_buckets = 1000 self.assertEqual(seeded_experiment._calculate_bucket("t2_1"), int(595))
def _get_experiment(self, name: str) -> Optional[Experiment]: if name not in self._experiment_cache: experiment_config = self._get_config(name) if not experiment_config: experiment = None else: try: experiment = parse_experiment(experiment_config) except Exception as err: logger.error("Invalid configuration for experiment %s: %s", name, err) return None self._experiment_cache[name] = experiment return self._experiment_cache[name]
def test_user_in(self): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": { "user_name": { "Gary": "active", "dave": "active", "ALL_UPPERCASE": "active" } }, "variants": { "active": 0 }, }, } feature_flag = parse_experiment(cfg) self.assertEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, user_name="Gary"), "active", ) self.assertEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, user_name=self.user_name), "active", ) all_uppercase_id = "t2_f00d" all_uppercase_name = "ALL_UPPERCASE" self.assertEqual( feature_flag.variant(user_id=all_uppercase_id, logged_in=True, user_name=all_uppercase_name), "active", ) self.assertEqual( feature_flag.variant(user_id=all_uppercase_id, logged_in=True, user_name=all_uppercase_name.lower()), "active", )
def test_no_version_allowed(self): cfg = { "id": 1, "name": "test", "owner": "test", "type": "r2", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { "control_1": 10, "control_2": 10 } }, } experiment = parse_experiment(cfg) self.assertTrue(isinstance(experiment, R2Experiment)) self.assertTrue(experiment.should_log_bucketing()) self.assertIs(experiment.version, None)
def test_calculate_bucket(self): cfg = { "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { "control_1": 10, "control_2": 10 } }, } experiment = parse_experiment(cfg) # Give ourselves enough users that we can get some reasonable amount of # precision when checking amounts per bucket. num_users = experiment.num_buckets * 1000 fullnames = [] for i in range(num_users): fullnames.append("t2_%s" % str(i)) counter = collections.Counter() for fullname in fullnames: bucket = experiment._calculate_bucket(fullname) counter[bucket] += 1 # Ensure bucketing is deterministic. self.assertEqual(bucket, experiment._calculate_bucket(fullname)) for bucket in range(experiment.num_buckets): # We want an even distribution across buckets. expected = num_users / experiment.num_buckets actual = counter[bucket] # Calculating the percentage difference instead of looking at the # raw difference scales better as we change num_users. percent_equal = float(actual) / expected self.assertAlmostEqual(percent_equal, 1.0, delta=0.10, msg="bucket: %s" % bucket)
def test_r2_type_returns_r2_experiment(self): cfg = { "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { "control_1": 10, "control_2": 10 } }, } experiment = parse_experiment(cfg) self.assertTrue(isinstance(experiment, R2Experiment)) self.assertTrue(experiment.should_log_bucketing()) self.assertEqual(experiment.version, "1")
def _get_experiment(self, name: str) -> Optional[Experiment]: if name in self._global_cache: return self._global_cache[name] if self._cfg_data is None: warn_deprecated("config_watcher will be removed in Baseplate 2.0.") self._cfg_data = self._get_config() if name not in self._cfg_data: logger.info("Experiment <%r> not found in experiment config", name) return None try: experiment = parse_experiment(self._cfg_data[name]) self._global_cache[name] = experiment return experiment except Exception as err: logger.error("Invalid configuration for experiment %s: %s", name, err) return None
def test_after_expires_returns_forced_variant(self): expires = (datetime.now() - timedelta(days=30)).strftime(ISO_DATE_FMT) cfg = { "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "expires": expires, "enabled": True, "experiment": { "id": 1, "name": "test", "variants": { "control_1": 10, "control_2": 10 } }, } experiment = parse_experiment(cfg) self.assertTrue(isinstance(experiment, ForcedVariantExperiment))
def simulate_percent_loggedout(wanted_percent): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "targeting": { "logged_in": [False] }, "variants": { "active": wanted_percent }, }, } feature_flag = parse_experiment(cfg) return (feature_flag.variant(user_id="t2_%s" % str(i), logged_in=False) == "active" for i in range(num_users))
def test_before_start_ts_returns_forced_variant(self): cfg = { "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() + THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS * 2, "enabled": True, "experiment": { "id": 1, "name": "test", "variants": { "control_1": 10, "control_2": 10 } }, } experiment = parse_experiment(cfg) self.assertTrue(isinstance(experiment, ForcedVariantExperiment))
def test_calculate_bucket_value(self): experiment = create_simple_experiment() experiment.num_buckets = 1000 self.assertEqual(experiment._calculate_bucket("t2_1"), int(867)) seeded_cfg = { "id": 1, "name": "test_experiment", "owner": "test", "type": "single_variant", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "enabled": True, "experiment": { "variants": [ { "name": "variant_1", "size": 0.1 }, { "name": "variant_2", "size": 0.1 }, ], "experiment_version": 1, "shuffle_version": 1, "bucket_seed": "some new seed", }, } seeded_experiment = parse_experiment(seeded_cfg) self.assertNotEqual(seeded_experiment.seed, experiment.seed) self.assertIsNot(seeded_experiment.seed, None) seeded_experiment.num_buckets = 1000 self.assertEqual(seeded_experiment._calculate_bucket("t2_1"), int(924))
def test_unknown_type_returns_null_experiment(self): cfg = { "id": 1, "name": "test", "owner": "test", "type": "unknown", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "id": 1, "name": "test", "variants": { "control_1": 10, "control_2": 10 } }, } experiment = parse_experiment(cfg) self.assertTrue(isinstance(experiment, ForcedVariantExperiment)) self.assertIs(experiment.variant(), None) self.assertFalse(experiment.should_log_bucketing())
def test_bucket_val(self, choose_variant_mock): choose_variant_mock.return_value = "fake_variant" cfg = { "id": 1, "name": "test_experiment", "owner": "test", "type": "single_variant", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "enabled": True, "experiment": { "variants": [ { "name": "variant_1", "size": 0.5 }, { "name": "variant_2", "size": 0.5 }, ], "experiment_version": 1, "bucket_val": "new_bucket_val", }, } experiment = parse_experiment(cfg) experiment._choose_variant = choose_variants_override variant_default_bucket_val = experiment.variant(user_id="t2_1") variant_new_bucket_val = experiment.variant( new_bucket_val="some_value") self.assertIs(variant_default_bucket_val, None) self.assertIsNot(variant_new_bucket_val, None)
def test_get_override(self): exp_config = get_simple_config() override_config = get_simple_override_config() exp_config["experiment"]["overrides"] = override_config experiment_with_overrides = parse_experiment(exp_config) self.assertEqual( experiment_with_overrides.get_override(user_id="t2_1"), "override_variant_1") self.assertEqual( experiment_with_overrides.get_override(user_id="t2_2"), "override_variant_2") self.assertEqual( experiment_with_overrides.get_override(user_id="t2_3"), "override_variant_3") self.assertEqual( experiment_with_overrides.get_override(user_id="t2_4"), "override_variant_1")
def test_expires_ignores_start_ts(self): expires = (datetime.now() + timedelta(days=30)).strftime(ISO_DATE_FMT) cfg = { "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() + THIRTY_DAYS, "expires": expires, "enabled": True, "experiment": { "id": 1, "name": "test", "variants": { "control_1": 10, "control_2": 10 } }, } experiment = parse_experiment(cfg) self.assertFalse(isinstance(experiment, ForcedVariantExperiment))
def test_variant_call_with_overrides(self, choose_variant_mock): choose_variant_mock.return_value = "mocked_variant" exp_config = get_simple_config() override_config = get_simple_override_config() exp_config["experiment"]["overrides"] = override_config experiment_with_overrides = parse_experiment(exp_config) self.assertEqual(experiment_with_overrides.variant(user_id="t2_1"), "override_variant_1") self.assertEqual(experiment_with_overrides.variant(user_id="t2_2"), "override_variant_2") self.assertEqual(experiment_with_overrides.variant(user_id="t2_3"), "override_variant_3") self.assertEqual(experiment_with_overrides.variant(user_id="t2_4"), "override_variant_1") self.assertEqual(experiment_with_overrides.variant(user_id="t2_5"), "mocked_variant")
def test_non_string_override_value(self): experiment = parse_experiment({ "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": { "logged_in": { True: "active" } }, "variants": { "active": 10, "control_1": 10, "control_2": 20 }, }, }) variant = experiment.variant(logged_in=True) self.assertEqual(variant, "active")
def test_multiple(self): # is_admin, globally off should still be False cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "global_override": None, "experiment": { "overrides": { "user_roles": { "admin": "active" } }, "variants": { "active": 0 }, }, } feature_flag = parse_experiment(cfg) self.assertNotEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, user_roles=["admin"]), "active", ) # globally on but not admin should still be True cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "global_override": "active", "experiment": { "overrides": { "user_roles": { "admin": "active" } }, "variants": { "active": 0 }, }, } feature_flag = parse_experiment(cfg) self.assertEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, user_roles=["admin"]), "active", ) self.assertEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active") # no URL but admin should still be True cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "overrides": { "user_roles": { "admin": "active" }, "url_features": { "test_featurestate": "active" }, }, "variants": { "active": 0 }, }, } feature_flag = parse_experiment(cfg) self.assertEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in, user_roles=["admin"]), "active", ) self.assertEqual( feature_flag.variant( user_id=self.user_id, logged_in=self.user_logged_in, url_features=["test_featurestate"], ), "active", ) self.assertNotEqual( feature_flag.variant(user_id=self.user_id, logged_in=self.user_logged_in), "active")
def create_simple_experiment(): cfg = get_simple_config() experiment = parse_experiment(cfg) return experiment