def test_calculate_bucket(self): experiment = create_simple_experiment() # Give ourselves enough users that we can get some reasonable amount of # precision when checking amounts per bucket. num_users = experiment.num_buckets * 2000 fullnames = [] for i in range(num_users): fullnames.append("t2_%s" % str(i)) counter = collections.Counter() for fullname in fullnames: bucket = experiment._calculate_bucket(fullname) counter[bucket] += 1 # Ensure bucketing is deterministic. self.assertEqual(bucket, experiment._calculate_bucket(fullname)) for bucket in range(experiment.num_buckets): # We want an even distribution across buckets. expected = num_users / experiment.num_buckets actual = counter[bucket] # Calculating the percentage difference instead of looking at the # raw difference scales better as we change num_users. percent_equal = float(actual) / expected self.assertAlmostEqual(percent_equal, 1.0, delta=.10, msg='bucket: %s' % bucket)
def test_calculate_bucket_with_seed(self): cfg = { "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { "control_1": 10, "control_2": 10, }, "seed": "itscoldintheoffice", } } experiment = parse_experiment(cfg) # Give ourselves enough users that we can get some reasonable amount of # precision when checking amounts per bucket. num_users = experiment.num_buckets * 1000 fullnames = [] for i in range(num_users): fullnames.append("t2_%s" % str(i)) counter = collections.Counter() bucketing_changed = False for fullname in fullnames: self.assertEqual(experiment.seed, "itscoldintheoffice") bucket1 = experiment._calculate_bucket(fullname) counter[bucket1] += 1 # Ensure bucketing is deterministic. self.assertEqual(bucket1, experiment._calculate_bucket(fullname)) current_seed = experiment.seed experiment.seed = "newstring" bucket2 = experiment._calculate_bucket(fullname) experiment.seed = current_seed # check that the bucketing changed at some point. Can't compare # bucket1 to bucket2 inline because sometimes the user will fall # into both buckets, and test will fail if bucket1 != bucket2: bucketing_changed = True self.assertTrue(bucketing_changed) for bucket in range(experiment.num_buckets): # We want an even distribution across buckets. expected = num_users / experiment.num_buckets actual = counter[bucket] # Calculating the percentage difference instead of looking at the # raw difference scales better as we change NUM_USERS. percent_equal = float(actual) / expected self.assertAlmostEqual(percent_equal, 1.0, delta=.10, msg='bucket: %s' % bucket)
def test_change_shuffle_version_changes_bucketing(self): cfg = get_simple_config() experiment_version_1 = parse_experiment(cfg) shuffle_cfg = get_simple_config() shuffle_cfg['experiment']['shuffle_version'] = 2 experiment_version_2 = parse_experiment(shuffle_cfg) # Give ourselves enough users that we can get some reasonable amount of # precision when checking amounts per bucket. num_users = experiment_version_1.num_buckets * 100 fullnames = [] for i in range(num_users): fullnames.append("t2_%s" % str(i)) counter = collections.Counter() bucketing_changed = False for fullname in fullnames: bucket1 = experiment_version_1._calculate_bucket(fullname) counter[bucket1] += 1 # Ensure bucketing is deterministic. self.assertEqual(bucket1, experiment_version_1._calculate_bucket(fullname)) bucket2 = experiment_version_2._calculate_bucket(fullname) # check that the bucketing changed at some point. Can't compare # bucket1 to bucket2 inline because sometimes the user will fall # into both buckets, and test will fail. When a user doesn't match, # break out of loop if bucket1 != bucket2: bucketing_changed = True break self.assertTrue(bucketing_changed)
def test_distribution_def_odd(self): variant_cfg = generate_variant_config() variant_cfg.append({"name": "variant_4", "size": 0.25}) variant_set = MultiVariantSet( variants=variant_cfg, num_buckets=NUM_BUCKETS_ODD, ) variant_counts = { "variant_1": 0, "variant_2": 0, "variant_3": 0, "variant_4": 0, None: 0, } for bucket in range(0, NUM_BUCKETS_ODD): variant = variant_set.choose_variant(bucket) variant_counts[variant] += 1 self.assertEqual(len(variant_counts), 5) self.assertEqual(variant_counts["variant_1"], 259) self.assertEqual(variant_counts["variant_2"], 259) self.assertEqual(variant_counts["variant_3"], 259) self.assertEqual(variant_counts["variant_4"], 259) self.assertEqual(variant_counts[None], 1)
def test_distribution_def_odd(self): variant_cfg = [ { "name": "variant_1", "size": 0.5 }, { "name": "variant_2", "size": 0.5 }, ] variant_set = SingleVariantSet( variants=variant_cfg, num_buckets=NUM_BUCKETS_ODD, ) variant_counts = { "variant_1": 0, "variant_2": 0, None: 0, } for bucket in range(0, NUM_BUCKETS_ODD): variant = variant_set.choose_variant(bucket) variant_counts[variant] += 1 self.assertEqual(len(variant_counts), 3) self.assertEqual(variant_counts["variant_1"], 518) self.assertEqual(variant_counts["variant_2"], 518) self.assertEqual(variant_counts[None], 1)
def test_distribution_single_bucket(self): cfg = [ { "name": "variant_1", "size": 0.001 }, { "name": "variant_2", "size": 0 }, ] variant_set = SingleVariantSet(variants=cfg, num_buckets=NUM_BUCKETS_DEFAULT) variant_counts = { "variant_1": 0, "variant_2": 0, None: 0, } for bucket in range(0, NUM_BUCKETS_DEFAULT): variant = variant_set.choose_variant(bucket) variant_counts[variant] += 1 self.assertEqual(len(variant_counts), 3) self.assertEqual(variant_counts['variant_1'], 1) self.assertEqual(variant_counts['variant_2'], 0) self.assertEqual(variant_counts[None], 999)
def test_calculate_bucket(self): cfg = { "id": 1, "name": "test", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { "control_1": 10, "control_2": 10, } } } experiment = parse_experiment(cfg) # Give ourselves enough users that we can get some reasonable amount of # precision when checking amounts per bucket. num_users = experiment.num_buckets * 1000 fullnames = [] for i in range(num_users): fullnames.append("t2_%s" % str(i)) counter = collections.Counter() for fullname in fullnames: bucket = experiment._calculate_bucket(fullname) counter[bucket] += 1 # Ensure bucketing is deterministic. self.assertEqual(bucket, experiment._calculate_bucket(fullname)) for bucket in range(experiment.num_buckets): # We want an even distribution across buckets. expected = num_users / experiment.num_buckets actual = counter[bucket] # Calculating the percentage difference instead of looking at the # raw difference scales better as we change num_users. percent_equal = float(actual) / expected self.assertAlmostEqual(percent_equal, 1.0, delta=.10, msg='bucket: %s' % bucket)
def get_users(num_users, logged_in=True): users = [] for i in range(num_users): if logged_in: name = str(i) else: name = None users.append(dict( name=name, id="t2_%s" % str(i), logged_in=logged_in, )) return users
def generate_content(num_content, content_type): content = [] if content_type == "subreddit": id_fmt = "t5_%s" elif content_type == "link": id_fmt = "t3_%s" elif content_type == "comment": id_fmt = "t1_%s" else: raise ValueError("Unknown content type: %s", content_type) for i in range(num_content): content.append(dict(id=id_fmt % i, type=content_type)) return content
def test_distribution_def_buckets(self): variant_set = create_rollout_variant_set() variant_counts = { "variant_1": 0, None: 0, } for bucket in range(0, NUM_BUCKETS_DEFAULT): variant = variant_set.choose_variant(bucket) variant_counts[variant] += 1 self.assertEqual(len(variant_counts), 2) self.assertEqual(variant_counts['variant_1'], 250) self.assertEqual(variant_counts[None], 750)
def test_distribution_def_buckets(self): variant_set = create_multi_variant_set() variant_counts = { "variant_1": 0, "variant_2": 0, "variant_3": 0, None: 0, } for bucket in range(0, NUM_BUCKETS_DEFAULT): variant = variant_set.choose_variant(bucket) variant_counts[variant] += 1 self.assertEqual(len(variant_counts), 4) for variant_count in variant_counts.values(): self.assertEqual(variant_count, 250)
def simulate_percent_loggedout(wanted_percent): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "expires": (datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT), "experiment": { "targeting": { "logged_in": [False], }, "variants": { "active": wanted_percent, }, }, } feature_flag = parse_experiment(cfg) return (feature_flag.variant( user_id="t2_%s" % str(i), logged_in=False, ) == "active" for i in range(num_users))
def simulate_percent_loggedout(wanted_percent): cfg = { "id": 1, "name": "test_feature", "type": "feature_flag", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "targeting": { "logged_in": [False], }, "variants": { "active": wanted_percent, }, }, } feature_flag = parse_experiment(cfg) return (feature_flag.variant( user_id="t2_%s" % str(i), logged_in=False, ) == "active" for i in range(num_users))
def test_distribution_def_odd(self): variant_cfg = [ { "name": "variant_1", "size": 1.0 }, ] variant_set = RolloutVariantSet( variants=variant_cfg, num_buckets=NUM_BUCKETS_ODD, ) variant_counts = { "variant_1": 0, None: 0, } for bucket in range(0, NUM_BUCKETS_ODD): variant = variant_set.choose_variant(bucket) variant_counts[variant] += 1 self.assertEqual(len(variant_counts), 2) self.assertEqual(variant_counts["variant_1"], 1037) self.assertEqual(variant_counts[None], 0)
def test_choose_variant(self): control_only = parse_experiment({ "id": 1, "name": "control_only", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { "control_1": 10, "control_2": 10, } } }) three_variants = parse_experiment({ "id": 1, "name": "three_variants", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { 'remove_vote_counters': 5, 'control_1': 10, 'control_2': 5, } } }) three_variants_more = parse_experiment({ "id": 1, "name": "three_variants_more", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { 'remove_vote_counters': 15.6, 'control_1': 10, 'control_2': 20, } } }) counters = collections.defaultdict(collections.Counter) for bucket in range(control_only.num_buckets): variant = control_only._choose_variant(bucket) if variant: counters[control_only.name][variant] += 1 # Ensure variant-choosing is deterministic. self.assertEqual(variant, control_only._choose_variant(bucket)) variant = three_variants._choose_variant(bucket) if variant: counters[three_variants.name][variant] += 1 # Ensure variant-choosing is deterministic. self.assertEqual(variant, three_variants._choose_variant(bucket)) previous_variant = variant variant = three_variants_more._choose_variant(bucket) if variant: counters[three_variants_more.name][variant] += 1 # Ensure variant-choosing is deterministic. self.assertEqual(variant, three_variants_more._choose_variant(bucket)) # If previously we had a variant, we should still have the same one # now. if previous_variant: self.assertEqual(variant, previous_variant) for experiment in (control_only, three_variants, three_variants_more): for variant, percentage in iteritems(experiment.variants): count = counters[experiment.name][variant] scaled_percentage = float(count) / (experiment.num_buckets / 100) self.assertEqual(scaled_percentage, percentage) # Test boundary conditions around the maximum percentage allowed for # variants. fifty_fifty = parse_experiment({ "id": 1, "name": "fifty_fifty", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { 'control_1': 50, 'control_2': 50, } } }) almost_fifty_fifty = parse_experiment({ "id": 1, "name": "almost_fifty_fifty", "owner": "test", "type": "r2", "version": "1", "start_ts": time.time() - THIRTY_DAYS, "stop_ts": time.time() + THIRTY_DAYS, "experiment": { "variants": { 'control_1': 49, 'control_2': 51, } } }) for bucket in range(fifty_fifty.num_buckets): for experiment in (fifty_fifty, almost_fifty_fifty): variant = experiment._choose_variant(bucket) counters[experiment.name][variant] += 1 count = counters[fifty_fifty.name]['control_1'] scaled_percentage = float(count) / (fifty_fifty.num_buckets / 100) self.assertEqual(scaled_percentage, 50) count = counters[fifty_fifty.name]['control_2'] scaled_percentage = float(count) / (fifty_fifty.num_buckets / 100) self.assertEqual(scaled_percentage, 50) count = counters[almost_fifty_fifty.name]['control_1'] scaled_percentage = float(count) / (almost_fifty_fifty.num_buckets / 100) self.assertEqual(scaled_percentage, 49) count = counters[almost_fifty_fifty.name]['control_2'] scaled_percentage = float(count) / (almost_fifty_fifty.num_buckets / 100) self.assertEqual(scaled_percentage, 50)