def test_change_shuffle_version_changes_bucketing(self):
cfg = get_simple_config()
experiment_version_1 = parse_experiment(cfg)
shuffle_cfg = get_simple_config()
shuffle_cfg['experiment']['shuffle_version'] = 2
experiment_version_2 = parse_experiment(shuffle_cfg)
# Give ourselves enough users that we can get some reasonable amount of
# precision when checking amounts per bucket.
num_users = experiment_version_1.num_buckets * 100
fullnames = []
for i in range(num_users):
fullnames.append("t2_%s" % str(i))
counter = collections.Counter()
bucketing_changed = False
for fullname in fullnames:
bucket1 = experiment_version_1._calculate_bucket(fullname)
counter[bucket1] += 1
# Ensure bucketing is deterministic.
self.assertEqual(bucket1,
experiment_version_1._calculate_bucket(fullname))
bucket2 = experiment_version_2._calculate_bucket(fullname)
# check that the bucketing changed at some point. Can't compare
# bucket1 to bucket2 inline because sometimes the user will fall
# into both buckets, and test will fail. When a user doesn't match,
# break out of loop
if bucket1 != bucket2:
bucketing_changed = True
break
self.assertTrue(bucketing_changed)
def test_subdomain_not_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"subdomain": {},
},
"variants": {
"active": 0,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
subdomain="beta",
), "active")
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
subdomain="",
), "active")
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"subdomain": {
"www": "active",
"betanauts": "active",
},
},
"variants": {
"active": 0,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
subdomain="beta",
), "active")
def test_subdomain_not_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"overrides": {
"subdomain": {},
},
"variants": {
"active": 0,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
subdomain="beta",
), "active")
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
subdomain="",
), "active")
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"overrides": {
"subdomain": {
"www": "active",
"betanauts": "active",
},
},
"variants": {
"active": 0,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
subdomain="beta",
), "active")
def test_user_not_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"user_name": {},
},
"variants": {
"active": 0,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
user_name=self.user_name,
), "active")
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"user_name": {
"dave": "active",
"joe": "active",
},
},
"variants": {
"active": 0,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
user_name=self.user_name,
), "active")
def test_subreddit_not_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"subreddit": {}
},
"variants": {
"active": 0
}
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
subreddit="wtf"),
"active",
)
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"subreddit": {
"wtfoobar": "active",
"aww": "active"
}
},
"variants": {
"active": 0
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
subreddit="wtf"),
"active",
)
def test_return_override_variant_without_bucket_val(self):
experiment = parse_experiment({
"id":
1,
"name":
"test",
"owner":
"test",
"type":
"r2",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"overrides": {
"user_name": {
"gary": "active",
},
},
"variants": {
'active': 10,
'control_1': 10,
'control_2': 20,
}
}
})
variant = experiment.variant(user_name="gary")
self.assertEqual(variant, "active")
variant = experiment.variant()
self.assertEqual(variant, None)
def test_variant_call_with_overrides(self, choose_variant_mock):
choose_variant_mock.return_value = "mocked_variant"
exp_config = get_simple_config()
override_config = get_simple_override_config()
exp_config['experiment']['overrides'] = override_config
experiment_with_overrides = parse_experiment(exp_config)
self.assertEqual(
experiment_with_overrides.variant(user_id='t2_1'),
'override_variant_1'
)
self.assertEqual(
experiment_with_overrides.variant(user_id='t2_2'),
'override_variant_2'
)
self.assertEqual(
experiment_with_overrides.variant(user_id='t2_3'),
'override_variant_3'
)
self.assertEqual(
experiment_with_overrides.variant(user_id='t2_4'),
'override_variant_1'
)
self.assertEqual(
experiment_with_overrides.variant(user_id='t2_5'),
'mocked_variant'
)
def test_get_override(self):
exp_config = get_simple_config()
override_config = get_simple_override_config()
exp_config['experiment']['overrides'] = override_config
experiment_with_overrides = parse_experiment(exp_config)
self.assertEqual(
experiment_with_overrides.get_override(user_id='t2_1'),
'override_variant_1'
)
self.assertEqual(
experiment_with_overrides.get_override(user_id='t2_2'),
'override_variant_2'
)
self.assertEqual(
experiment_with_overrides.get_override(user_id='t2_3'),
'override_variant_3'
)
self.assertEqual(
experiment_with_overrides.get_override(user_id='t2_4'),
'override_variant_1'
)
def test_newer_than_only_on_logged_in_check(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"expires": (datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"targeting": {
"logged_in": [True],
"user_name": ["gary"],
},
"newer_than": int(time.time()) + THIRTY_DAYS.total_seconds(),
"variants": {
"active": 100,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
user_name="gary",
), "active")
self.assertNotEqual(feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
), "active")
def test_is_not_newer_than(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"targeting": {
"logged_in": [True]
},
"newer_than": int(time.time()) + THIRTY_DAYS,
"variants": {
"active": 100
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
user_created=int(time.time())),
"active",
)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in), "active")
def test_url_disabled(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"url_features": {
"test_state": "active"
}
},
"variants": {
"active": 0
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in), "active")
self.assertNotEqual(
feature_flag.variant(user_id=self.user_id,
logged_in=self.user_logged_in,
url_features=["x"]),
"active",
)
def test_bucket_val(self, choose_variant_mock):
choose_variant_mock.return_value = "fake_variant"
cfg = {
"id": 1,
"name": "test_experiment",
"owner": "test",
"type": "single_variant",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"enabled": True,
"experiment": {
"variants": [
{"name": "variant_1", "size": 0.5},
{"name": "variant_2", "size": 0.5},
],
"experiment_version": 1,
"bucket_val": "new_bucket_val",
},
}
experiment = parse_experiment(cfg)
experiment._choose_variant = choose_variants_override
variant_default_bucket_val = experiment.variant(user_id="t2_1")
variant_new_bucket_val = experiment.variant(new_bucket_val="some_value")
self.assertIs(variant_default_bucket_val, None)
self.assertIsNot(variant_new_bucket_val, None)
def test_non_string_override_value(self):
experiment = parse_experiment({
"id":
1,
"name":
"test",
"owner":
"test",
"type":
"r2",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"overrides": {
"logged_in": {
True: "active",
},
},
"variants": {
'active': 10,
'control_1': 10,
'control_2': 20,
}
}
})
variant = experiment.variant(logged_in=True)
self.assertEqual(variant, "active")
def test_subreddit_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"overrides": {
"subreddit": {
"WTF": "active",
"aww": "active",
},
},
"variants": {
"active": 0,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
subreddit="WTF",
), "active")
self.assertEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
subreddit="wtf",
), "active")
def test_calculate_bucket_value(self):
experiment = create_simple_experiment()
experiment.num_buckets = 1000
self.assertEqual(experiment._calculate_bucket("t2_1"), int(867))
seeded_cfg = {
"id": 1,
"name": "test_experiment",
"owner": "test",
"type": "single_variant",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"enabled": True,
"experiment": {
"variants": [
{"name": "variant_1", "size": 0.1},
{"name": "variant_2", "size": 0.1},
],
"experiment_version": 1,
"shuffle_version": 1,
"bucket_seed": "some new seed",
},
}
seeded_experiment = parse_experiment(seeded_cfg)
self.assertNotEqual(seeded_experiment.seed, experiment.seed)
self.assertIsNot(seeded_experiment.seed, None)
seeded_experiment.num_buckets = 1000
self.assertEqual(seeded_experiment._calculate_bucket("t2_1"), int(924))
def _simulate_experiment(self, config, static_vars, target_var, targets):
num_experiments = len(targets)
counter = collections.Counter()
self.mock_filewatcher.get_data.return_value = {config["name"]: config}
for target in targets:
experiment_vars = {target_var: target}
experiment_vars.update(static_vars)
user = experiment_vars.pop("user")
content = experiment_vars.pop("content")
experiments = self.get_experiment_client("test")
variant = experiments.variant(config["name"],
user_id=user["id"],
user_name=user["name"],
logged_in=user["logged_in"],
content_id=content["id"],
content_type=content["type"],
**experiment_vars)
if variant:
counter[variant] += 1
# this test will still probabilistically fail, but we can mitigate
# the likeliness of that happening
error_bar_percent = 100. / math.sqrt(num_experiments)
experiment = parse_experiment(config)
for variant, percent in iteritems(experiment.variants):
# Our actual percentage should be within our expected percent
# (expressed as a part of 100 rather than a fraction of 1)
# +- 1%.
measured_percent = (float(counter[variant]) /
num_experiments) * 100
self.assertAlmostEqual(measured_percent,
percent,
delta=error_bar_percent)
def test_experiment_disabled(self):
experiments_cfg = {
"id": 1,
"name": "test_experiment",
"owner": "test",
"type": "single_variant",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"enabled": False,
"experiment": {
"variants": [
{
"name": "variant_1",
"size": 0.5,
},
{
"name": "variant_2",
"size": 0.5,
},
],
"experiment_version":
1,
}
}
experiment = parse_experiment(experiments_cfg)
variant = experiment.variant(user_id="t2_1")
self.assertIs(variant, None)
def test_newer_than_only_on_logged_in_check(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"targeting": {
"logged_in": [True],
"user_name": ["gary"],
},
"newer_than": int(time.time()) + THIRTY_DAYS,
"variants": {
"active": 100,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
user_name="gary",
), "active")
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
), "active")
def test_return_override_variant_without_bucket_val(self):
experiment = parse_experiment({
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"user_name": {
"gary": "active",
},
},
"variants": {
'active': 10,
'control_1': 10,
'control_2': 20,
}
}
})
variant = experiment.variant(user_name="gary")
self.assertEqual(variant, "active")
variant = experiment.variant()
self.assertEqual(variant, None)
def test_is_not_newer_than(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"newer_than": int(time.time()) + THIRTY_DAYS.total_seconds(),
"variants": {
"active": 100,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
user_created=int(time.time()),
), "active")
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
), "active")
def test_admin_enabled(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"user_roles": {
"admin": "active",
},
},
"variants": {
"active": 0,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
), "active")
self.assertEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
user_roles=["admin"],
), "active")
def test_admin_enabled(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"overrides": {
"user_groups": {
"admin": "active",
},
},
"variants": {
"active": 0,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertNotEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
), "active")
self.assertEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
user_groups=["admin"],
), "active")
def test_calculate_bucket_with_seed(self):
seeded_cfg = {
"id": 1,
"name": "test_experiment",
"owner": "test",
"type": "single_variant",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"enabled": True,
"experiment": {
"variants": [
{"name": "variant_1", "size": 0.1},
{"name": "variant_2", "size": 0.1},
],
"experiment_version": 1,
"shuffle_version": 1,
"bucket_seed": "some_new_seed",
},
}
experiment = parse_experiment(seeded_cfg)
# Give ourselves enough users that we can get some reasonable amount of
# precision when checking amounts per bucket.
num_users = experiment.num_buckets * 2000
fullnames = []
for i in range(num_users):
fullnames.append("t2_%s" % str(i))
counter = collections.Counter()
bucketing_changed = False
for fullname in fullnames:
self.assertEqual(experiment.seed, "some_new_seed")
bucket1 = experiment._calculate_bucket(fullname)
counter[bucket1] += 1
# Ensure bucketing is deterministic.
self.assertEqual(bucket1, experiment._calculate_bucket(fullname))
current_seed = experiment.seed
experiment._seed = "newstring"
bucket2 = experiment._calculate_bucket(fullname)
experiment._seed = current_seed
# check that the bucketing changed at some point. Can't compare
# bucket1 to bucket2 inline because sometimes the user will fall
# into both buckets, and test will fail.
if bucket1 != bucket2:
bucketing_changed = True
self.assertTrue(bucketing_changed)
for bucket in range(experiment.num_buckets):
# We want an even distribution across buckets.
expected = num_users / experiment.num_buckets
actual = counter[bucket]
# Calculating the percentage difference instead of looking at the
# raw difference scales better as we change NUM_USERS.
percent_equal = float(actual) / expected
self.assertAlmostEqual(percent_equal, 1.0, delta=0.10, msg="bucket: %s" % bucket)
def test_calculate_bucket_with_seed(self):
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"variants": {
"control_1": 10,
"control_2": 10,
},
"seed": "itscoldintheoffice",
}
}
experiment = parse_experiment(cfg)
# Give ourselves enough users that we can get some reasonable amount of
# precision when checking amounts per bucket.
num_users = experiment.num_buckets * 1000
fullnames = []
for i in range(num_users):
fullnames.append("t2_%s" % str(i))
counter = collections.Counter()
bucketing_changed = False
for fullname in fullnames:
self.assertEqual(experiment.seed, "itscoldintheoffice")
bucket1 = experiment._calculate_bucket(fullname)
counter[bucket1] += 1
# Ensure bucketing is deterministic.
self.assertEqual(bucket1, experiment._calculate_bucket(fullname))
current_seed = experiment.seed
experiment.seed = "newstring"
bucket2 = experiment._calculate_bucket(fullname)
experiment.seed = current_seed
# check that the bucketing changed at some point. Can't compare
# bucket1 to bucket2 inline because sometimes the user will fall
# into both buckets, and test will fail
if bucket1 != bucket2:
bucketing_changed = True
self.assertTrue(bucketing_changed)
for bucket in range(experiment.num_buckets):
# We want an even distribution across buckets.
expected = num_users / experiment.num_buckets
actual = counter[bucket]
# Calculating the percentage difference instead of looking at the
# raw difference scales better as we change NUM_USERS.
percent_equal = float(actual) / expected
self.assertAlmostEqual(percent_equal,
1.0,
delta=.10,
msg='bucket: %s' % bucket)
def test_calculate_bucket_value(self):
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"control_1": 10,
"control_2": 10,
}
}
}
experiment = parse_experiment(cfg)
experiment.num_buckets = 1000
self.assertEqual(experiment._calculate_bucket("t2_1"), long(236))
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"seed": "test-seed",
"variants": {
"control_1": 10,
"control_2": 10,
}
}
}
seeded_experiment = parse_experiment(cfg)
self.assertNotEqual(seeded_experiment.seed, experiment.seed)
self.assertIsNot(seeded_experiment.seed, None)
seeded_experiment.num_buckets = 1000
self.assertEqual(
seeded_experiment._calculate_bucket("t2_1"),
long(595),
)
def test_targeting_in_config(self):
cfg = get_simple_config()
targeting_cfg = get_targeting_config()
cfg["experiment"]["targeting"] = targeting_cfg
experiment_with_targeting = parse_experiment(cfg)
self.assertTrue(
experiment_with_targeting.is_targeted(is_mod=True, is_logged_in=True, random_numeric=5)
)
def test_variant_returns_none_if_out_of_time_window(self, choose_variant_mock):
choose_variant_mock.return_value = 'fake_variant'
valid_cfg = get_simple_config()
experiment_valid = parse_experiment(valid_cfg)
expired_cfg = get_simple_config()
expired_cfg['stop_ts'] = time.time() - FIVE_DAYS
experiment_expired = parse_experiment(expired_cfg)
experiment_not_started_cfg = get_simple_config()
experiment_not_started_cfg['start_ts'] = time.time() + FIVE_DAYS
experiment_not_started = parse_experiment(experiment_not_started_cfg)
variant_valid = experiment_valid.variant(user_id="t2_1")
variant_expired = experiment_expired.variant(user_id="t2_1")
variant_not_started = experiment_not_started.variant(user_id="t2_1")
self.assertIsNot(variant_valid, None)
self.assertIs(variant_expired, None)
self.assertIs(variant_not_started, None)
def test_calculate_bucket_value(self):
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"variants": {
"control_1": 10,
"control_2": 10,
}
}
}
experiment = parse_experiment(cfg)
experiment.num_buckets = 1000
self.assertEqual(experiment._calculate_bucket("t2_1"), long(236))
cfg = {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"seed": "test-seed",
"variants": {
"control_1": 10,
"control_2": 10,
}
}
}
seeded_experiment = parse_experiment(cfg)
self.assertNotEqual(seeded_experiment.seed, experiment.seed)
self.assertIsNot(seeded_experiment.seed, None)
seeded_experiment.num_buckets = 1000
self.assertEqual(
seeded_experiment._calculate_bucket("t2_1"),
long(595),
)
def _get_experiment(self, name):
if name not in self._experiment_cache:
experiment_config = self._get_config(name)
if not experiment_config:
experiment = None
else:
try:
experiment = parse_experiment(experiment_config)
except Exception as err:
logger.error("Invalid configuration for experiment %s: %s",
name, err)
return None
self._experiment_cache[name] = experiment
return self._experiment_cache[name]
def test_user_in(self):
cfg = {
"id": 1,
"name": "test_feature",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"overrides": {
"user_name": {
"Gary": "active",
"dave": "active",
"ALL_UPPERCASE": "active",
},
},
"variants": {
"active": 0,
},
},
}
feature_flag = parse_experiment(cfg)
self.assertEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
user_name="Gary",
), "active")
self.assertEqual(
feature_flag.variant(
user_id=self.user_id,
logged_in=self.user_logged_in,
user_name=self.user_name,
), "active")
all_uppercase_id = "t2_f00d"
all_uppercase_name = "ALL_UPPERCASE"
self.assertEqual(
feature_flag.variant(
user_id=all_uppercase_id,
logged_in=True,
user_name=all_uppercase_name,
), "active")
self.assertEqual(
feature_flag.variant(
user_id=all_uppercase_id,
logged_in=True,
user_name=all_uppercase_name.lower(),
), "active")
