def test(self, data, control, variations): update = {} buckets = [control] + variations for bucket in buckets: total = data[bucket][0]['count'] successes = data[bucket][1]['count'] update[bucket] = (successes, total - successes) test = Trials(buckets) test.update(update) dominances = test.evaluate('dominance', control=control) lifts = test.evaluate('expected lift', control=control) intervals = test.evaluate('lift CI', control=control, level=95) for variation in variations: print('Variation {name}:'.format(name=variation)) s, t = update[variation] print('* Successes: {0}, Total: {1}, Percent: {2:2.3f}%'.format(s, t, 100*float(s)/t)) print('* E[lift] = {value:.2%}'.format(value=lifts[variation])) print('* P({lower:.2%} < lift < {upper:.2%}) = 95%'.format(lower=intervals[variation][0], upper=intervals[variation][2])) print('* P({name} > {control}) = {value:.2%}'.format(name=variation, control=control, value=dominances[variation])) print
"""Sociological poll example.""" import sys sys.path.append('..') from trials import Trials if __name__ == '__main__': test = Trials(['Poroshenko', 'Tymoshenko']) test.update({'Poroshenko': (48, 52), 'Tymoshenko': (12, 88)}) estimates = test.evaluate('posterior CI') dominance = test.evaluate('dominance', control='Tymoshenko') print('Poroshenko estimated vote share: {lower:.2%} - {upper:.2%} ' '(95% credibility)'.format(lower=estimates['Poroshenko'][0], upper=estimates['Poroshenko'][2])) print('Tymoshenko estimated vote share: {lower:.2%} - {upper:.2%} ' '(95% credibility)'.format(lower=estimates['Tymoshenko'][0], upper=estimates['Tymoshenko'][2])) print('Chance that Poroshenko beats Tymoshenko based on the poll data: ' '{chance:.2%}'.format(chance=dominance['Poroshenko']))
import sys sys.path.append('..') from trials import Trials if __name__ == '__main__': test = Trials(['A', 'B', 'C']) test.update({ 'A': (50, 10), 'B': (75, 15), 'C': (20, 15) }) lift = test.evaluate('lift', control='A') domination = test.evaluate('domination', control='A') for variation in ['B', 'C']: print('Variation {name}:'.format(name=variation)) print('* lift = {value:.2%}'.format(value=lift[variation])) print('* P({name} > {control}) = {value:.2%}' \ .format(name=variation, control='A', value=domination[variation]))
import sys sys.path.append('..') from trials import Trials if __name__ == '__main__': test = Trials(['Poroshenko', 'Tymoshenko']) test.update({ 'Poroshenko': (48, 52), 'Tymoshenko': (12, 88) }) estimates = test.evaluate('posterior CI') dominance = test.evaluate('dominance', control='Tymoshenko') print('Poroshenko estimated vote share: {lower:.2%} - {upper:.2%} ' '(95% credibility)' .format(lower=estimates['Poroshenko'][0], upper=estimates['Poroshenko'][2])) print('Tymoshenko estimated vote share: {lower:.2%} - {upper:.2%} ' '(95% credibility)' .format(lower=estimates['Tymoshenko'][0], upper=estimates['Tymoshenko'][2])) print('Chance that Poroshenko beats Tymoshenko based on the poll data: ' '{chance:.2%}'.format(chance=dominance['Poroshenko']))
"""Simple A/B test example.""" import sys sys.path.append('..') from trials import Trials if __name__ == '__main__': test = Trials(['A', 'B', 'C']) test.update({ 'A': (50, 10), 'B': (75, 15), 'C': (20, 15) }) dominance = test.evaluate('dominance', control='A') lift = test.evaluate('expected lift', control='A') interval = test.evaluate('lift CI', control='A', level=95) for variation in ['B', 'C']: print('Variation {name}:'.format(name=variation)) print('* E[lift] = {value:.2%}'.format(value=lift[variation])) print('* P({lower:.2%} < lift < {upper:.2%}) = 95%' .format(lower=interval[variation][0], upper=interval[variation][2])) print('* P({name} > {control}) = {value:.2%}' .format(name=variation, control='A', value=dominance[variation]))