def test_generally_random(self): items = ('True', 'False') weights = (1, 1) results = {} total = 50000 for _ in range(total): v = next(random_variant(items, weights)) results.setdefault(v, 0) results[v] += 1 a = results['True'] / float(total) b = results['False'] / float(total) assert a > .49 and a < .51 assert b > .49 and b < .51
def test_generally_random(self): items = ("True", "False") weights = (1, 1) results = {} total = 50000 for _ in range(total): v = next(random_variant(items, weights)) results.setdefault(v, 0) results[v] += 1 a = results["True"] / float(total) b = results["False"] / float(total) assert a > 0.49 and a < 0.51 assert b > 0.49 and b < 0.51
def test_generally_random_with_weights(self): items = ('A', 'B', 'C') weights = (1, 3, 6) results = {} total = 50000 for _ in range(total): v = next(random_variant(items, weights)) results.setdefault(v, 0) results[v] += 1 a = results['A'] / float(total) b = results['B'] / float(total) c = results['C'] / float(total) assert a > .09 and a < .11 assert b > .29 and b < .31 assert c > .59 and c < .61
def test_generally_random_with_weights(self): items = ("A", "B", "C") weights = (1, 3, 6) results = {} total = 50000 for _ in range(total): v = next(random_variant(items, weights)) results.setdefault(v, 0) results[v] += 1 a = results["A"] / float(total) b = results["B"] / float(total) c = results["C"] / float(total) assert a > 0.09 and a < 0.11 assert b > 0.29 and b < 0.31 assert c > 0.59 and c < 0.61
def split(self, experiment_name, *variants): """ Used to split and track user experience amongst one or more variants. :param experiment_name a unique string name for the experiment :param *variants can take many forms, depending on usage. Variants should be provided as arbitrary tuples in the format ('unique_string_label', any_value), ... e.g., >>> split('text_color', ('red', '#F00'), ('blue', '#00F')) ...where the first variant (in this example, 'red') represents the control and any additional variants represent alternatives. By default, variants are chosen with equal weight. You can tip the scales if you like by passing a proportional *integer* weight as the third element in each variant tuple: >>> split('text_color', ('red', '#F00', 2), ('blue', '#00F', 4)) Optionally, variants may be excluded entirely to fall back to a simple True/False 50/50 split, where True is the control and False is the experiment, e.g., >>> sidebar() if split('include_sidebar') else empty() """ # Perform some minimal type checking if not isinstance(experiment_name, string_types): raise RuntimeError( 'Invalid experiment name: %s must be a string.' % experiment_name) keys, values, weights = self._parse_variants(variants) b = self._backend # Record the experiment if it doesn't exist already experiment = b.get_experiment(experiment_name, keys) # If the current visitor hasn't been verified as a human, and we've not # required human verification, go ahead and mark them as a human. if self.count_humans_only is False and self.human is not True: b.mark_human(self.identity) if experiment is None: b.save_experiment(experiment_name, keys) experiment = b.get_experiment(experiment_name, keys) else: if set(experiment.variants) != set(keys): raise RuntimeError( 'An experiment named %s already exists with different ' 'variants.' % experiment_name) # Retrieve the variant assigned to the current user if experiment.name in self._environ.get('cleaver.override', {}): variant = self._environ['cleaver.override'][experiment.name] else: variant = b.get_variant(self.identity, experiment.name) if variant is None: # ...or choose (and store) one randomly if it doesn't exist yet variant = next(util.random_variant(keys, weights)) b.participate(self.identity, experiment.name, variant) return dict(zip(keys, values))[variant]
def split(self, experiment_name, *variants): """ Used to split and track user experience amongst one or more variants. :param experiment_name a unique string name for the experiment :param *variants can take many forms, depending on usage. Variants should be provided as arbitrary tuples in the format ('unique_string_label', any_value), ... e.g., >>> split('text_color', ('red', '#F00'), ('blue', '#00F')) ...where the first variant (in this example, 'red') represents the control and any additional variants represent alternatives. By default, variants are chosen with equal weight. You can tip the scales if you like by passing a proportional *integer* weight as the third element in each variant tuple: >>> split('text_color', ('red', '#F00', 2), ('blue', '#00F', 4)) Optionally, variants may be excluded entirely to fall back to a simple True/False 50/50 split, where True is the control and False is the experiment, e.g., >>> sidebar() if split('include_sidebar') else empty() """ # Perform some minimal type checking if not isinstance(experiment_name, string_types): raise RuntimeError( 'Invalid experiment name: %s must be a string.' % experiment_name ) keys, values, weights = self._parse_variants(variants) b = self._backend # Record the experiment if it doesn't exist already experiment = b.get_experiment(experiment_name, keys) # If the current visitor hasn't been verified as a human, and we've not # required human verification, go ahead and mark them as a human. if self.count_humans_only is False and self.human is not True: b.mark_human(self.identity) if experiment is None: b.save_experiment(experiment_name, keys) experiment = b.get_experiment(experiment_name, keys) else: if set(experiment.variants) != set(keys): raise RuntimeError( 'An experiment named %s already exists with different ' 'variants.' % experiment_name ) # Retrieve the variant assigned to the current user if experiment.name in self._environ.get('cleaver.override', {}): variant = self._environ['cleaver.override'][experiment.name] else: variant = b.get_variant(self.identity, experiment.name) if variant is None: # ...or choose (and store) one randomly if it doesn't exist yet variant = next(util.random_variant(keys, weights)) b.participate(self.identity, experiment.name, variant) return dict(zip(keys, values))[variant]