def test_bulk_athena(): # Same as Minerva (that is, delta = infinity) # Ballot-by-ballot Minerva should yield identical stopping rules to BRAVO. contest = Contest(100000, { 'A': 60000, 'B': 40000 }, 1, ['A'], ContestType.MAJORITY) athena = Athena(.1, 2**31 - 1, .01, contest) athena.compute_all_min_winner_ballots(athena.sub_audits['A-B']) # p0 not hardcoded as .5 for scalability with odd total contest ballots. p0 = (athena.contest.contest_ballots // 2) / athena.contest.contest_ballots log_winner_multiplier = math.log( athena.sub_audits['A-B'].sub_contest.winner_prop / p0) log_loser_multiplier = math.log( (1 - athena.sub_audits['A-B'].sub_contest.winner_prop) / p0) log_rhs = math.log(1 / athena.alpha) for i in range(len(athena.rounds)): n = athena.rounds[i] kmin = athena.sub_audits['A-B'].min_winner_ballots[i] # Assert this kmin satisfies ratio, but a kmin one less does not. assert kmin * log_winner_multiplier + ( n - kmin) * log_loser_multiplier > log_rhs assert (kmin - 1) * log_winner_multiplier + ( n - kmin + 1) * log_loser_multiplier <= log_rhs
def test_simple_athena(): simple_athena = Athena(.1, 2**31 - 1, .1, default_contest) assert simple_athena.alpha == .1 assert simple_athena.beta == 0.0 assert simple_athena.delta == 2**31 - 1 assert simple_athena.max_fraction_to_draw == .1 assert len(simple_athena.rounds) == 0 assert len(simple_athena.sub_audits['a-b'].min_winner_ballots) == 0 assert simple_athena.get_risk_level() is None
def test_athena_minerva_paper(): contest = Contest(100000, { 'A': 75000, 'B': 25000 }, 1, ['A'], ContestType.MAJORITY) athena = Athena(.1, 1, .1, contest) minerva = Minerva(.1, .1, contest) athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [50]) minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [50]) # From Athena paper assert athena.sub_audits['A-B'].min_winner_ballots == [32] assert minerva.sub_audits['A-B'].min_winner_ballots == [31]
def input_audit(contest: Contest, alpha: float = None, max_fraction_to_draw: float = None, audit_type: str = None, delta: float = None) -> Audit: # Create an audit from user-input. click.echo('\nCreate a new Audit') click.echo('==================\n') if alpha is None: alpha = click.prompt('Enter the desired risk limit', type=click.FloatRange(0.0, 1.0)) if max_fraction_to_draw is None: max_fraction_to_draw = click.prompt('Enter the maximum fraction of contest ballots to draw', type=click.FloatRange(0.0, 1.0)) if audit_type is None: audit_type = click.prompt('Select an audit type', type=audit_types) if delta is None and audit_type == 'athena': delta = click.prompt('Enter the Athena delta value', type=click.FloatRange(0.0)) if audit_type == 'brla': return BRLA(alpha, max_fraction_to_draw, contest) elif audit_type == 'minerva': return Minerva(alpha, max_fraction_to_draw, contest) elif audit_type == 'athena': return Athena(alpha, delta, max_fraction_to_draw, contest) # TODO: add creation for other types of audits. return None
def test_athena_execute_round(): contest = Contest(100000, { 'A': 75000, 'B': 25000 }, 1, ['A'], ContestType.MAJORITY) athena = Athena(.1, 1, .1, contest) assert not athena.execute_round(50, {'A': 31, 'B': 19}) assert not athena.stopped assert athena.sample_ballots['A'] == [31] assert athena.sample_ballots['B'] == [19] assert not athena.sub_audits['A-B'].stopped assert athena.rounds == [50] assert athena.execute_round(100, {'A': 70, 'B': 30}) assert athena.stopped assert athena.sample_ballots['A'] == [31, 70] assert athena.sample_ballots['B'] == [19, 30] assert athena.sub_audits['A-B'].stopped assert athena.rounds == [50, 100] assert athena.get_risk_level() < 0.1
def __init__(self, alpha, delta, reported, sample_size, db_mode=True, db_host='localhost', db_name='r2b2', db_port=27017, user='******', pwd='icanwrite', *args, **kwargs): super().__init__('athena', alpha, reported, 'tie', True, db_mode, db_host, db_port, db_name, user, pwd, *args, **kwargs) self.sample_size = sample_size self.total_relevant_ballots = sum(self.reported.tally.values()) # FIXME: temporary until pairwise contest fix is implemented self.contest_ballots = self.reported.contest_ballots self.reported.contest_ballots = self.total_relevant_ballots self.reported.winner_prop = self.reported.tally[ self.reported.reported_winners[0]] / self.reported.contest_ballots self.delta = delta self.audit = Athena(self.alpha, self.delta, 1.0, self.reported) if sample_size < self.audit.min_sample_size: raise ValueError( 'Sample size is less than minimum sample size for audit.') # FIXME: sorted candidate list will be created by new branch, update once merged # Generate a sorted underlying vote distribution sorted_tally = sorted(self.reported.tally.items(), key=lambda x: x[1], reverse=True) self.vote_dist = [(sorted_tally[0][0], self.total_relevant_ballots // 2)] for i in range(1, len(sorted_tally)): self.vote_dist.append( (sorted_tally[i][0], self.total_relevant_ballots)) self.vote_dist.append(('invalid', self.contest_ballots))
class AthenaOneRoundRisk(Simulation): """Simulate a 1-round Athena audit for a given sample size to compute risk limit.""" delta: float sample_size: int total_relevant_ballots: int vote_dist: List[Tuple[str, int]] audit: Athena def __init__(self, alpha, delta, reported, sample_size, db_mode=True, db_host='localhost', db_name='r2b2', db_port=27017, user='******', pwd='icanwrite', *args, **kwargs): super().__init__('athena', alpha, reported, 'tie', True, db_mode, db_host, db_port, db_name, user, pwd, *args, **kwargs) self.sample_size = sample_size self.total_relevant_ballots = sum(self.reported.tally.values()) # FIXME: temporary until pairwise contest fix is implemented self.contest_ballots = self.reported.contest_ballots self.reported.contest_ballots = self.total_relevant_ballots self.reported.winner_prop = self.reported.tally[ self.reported.reported_winners[0]] / self.reported.contest_ballots self.delta = delta self.audit = Athena(self.alpha, self.delta, 1.0, self.reported) if sample_size < self.audit.min_sample_size: raise ValueError( 'Sample size is less than minimum sample size for audit.') # FIXME: sorted candidate list will be created by new branch, update once merged # Generate a sorted underlying vote distribution sorted_tally = sorted(self.reported.tally.items(), key=lambda x: x[1], reverse=True) self.vote_dist = [(sorted_tally[0][0], self.total_relevant_ballots // 2)] for i in range(1, len(sorted_tally)): self.vote_dist.append( (sorted_tally[i][0], self.total_relevant_ballots)) self.vote_dist.append(('invalid', self.contest_ballots)) def trial(self, seed): """Execute a 1-round athena audit (using r2b2.athena.Athena)""" r.seed(seed) # Draw a sample of a given size sample = [0 for i in range(len(self.vote_dist))] for i in range(self.sample_size): ballot = r.randint(1, self.contest_ballots) for j in range(len(sample)): if ballot <= self.vote_dist[j][1]: sample[j] += 1 break relevant_sample_size = self.sample_size - sample[-1] # Perform audit computations self.audit._reset() self.audit.rounds.append(relevant_sample_size) self.audit.current_dist_null() self.audit.current_dist_reported() point_null = self.audit.distribution_null[sample[0]] point_reported = self.audit.distribution_reported_tally[sample[0]] p_value = self.audit.compute_risk(sample[0], relevant_sample_size) if p_value <= self.alpha and self.delta * point_reported > point_null: stop = True else: stop = False return { 'stop': stop, 'p_value': p_value, 'delta_computed': (point_null / point_reported), 'sample_size': self.sample_size, 'relevant_sample_size': relevant_sample_size, 'winner_ballots': sample[0] } def analyze(self, verbose: bool = False, hist: bool = False): """Analyze trials to get experimental risk. Args: verbose (bool): If true, analyze will print simulation analysis information. hist (bool): If true, analyze will generate and display 2 histograms: winner ballots found in the sample size and computed risk. """ if self.db_mode: trials = self.db.trial_lookup(self.sim_id) else: trials = self.trials num_trials = 0 stopped = 0 total_risk = 0 total_delta = 0 total_relevant_sampled = 0 winner_ballot_dist = [] risk_dist = [] delta_dist = [] for trial in trials: num_trials += 1 if trial['stop']: stopped += 1 total_relevant_sampled += trial['relevant_sample_size'] winner_ballot_dist.append(trial['winner_ballots']) total_risk += trial['p_value'] risk_dist.append(trial['p_value']) total_delta += trial['delta_computed'] delta_dist.append(trial['delta_computed']) if verbose: print('Analysis\n========') print('Underlying election is tied\n') print('Number of trials: {}'.format(num_trials)) print('Number of stopped: {}'.format(stopped)) print('Risk Limit: {:%}'.format(self.alpha)) print('Risk Computed: {:%}'.format(stopped / num_trials)) print('Delta Condition: {}'.format(self.delta)) print('Avg. Delta Computed: {}'.format(total_delta / num_trials)) if hist: histogram( winner_ballot_dist, 'Winner ballots found in sample of size: {}'.format( self.sample_size)) histogram(risk_dist, 'Risk (p_value) dist.') histogram(delta_dist, 'Delta (computed) dist.') # Update simulation entry to include analysis if self.db_mode: self.db.update_analysis(self.sim_id, (stopped / num_trials)) return stopped / num_trials
class AthenaOneRoundStoppingProb(Simulation): """Simulate a 1-round Athena audit for a given sample size to compute stopping probability.""" delta: float sample_size: int total_relevant_ballots: int vote_dist: List[Tuple[str, int]] audit: Athena def __init__(self, alpha, delta, reported, sample_size, db_mode=True, db_host='localhost', db_name='r2b2', db_port=27017, user='******', pwd='icanwrite', *args, **kwargs): super().__init__('athena', alpha, reported, 'reported', True, db_mode, db_host, db_port, db_name, user, pwd, *args, **kwargs) self.delta = delta self.sample_size = sample_size self.total_relevant_ballots = sum(self.reported.tally.values()) # FIXME: temporary until pairwise contest fix is implemented self.contest_ballots = self.reported.contest_ballots self.reported.contest_ballots = self.total_relevant_ballots self.reported.winner_prop = self.reported.tally[ self.reported.reported_winners[0]] / self.reported.contest_ballots self.audit = Athena(self.alpha, self.delta, 1.0, self.reported) if sample_size < self.audit.min_sample_size: raise ValueError( 'Sample size is less than minimum sample size for audit') # FIXME: sorted candidate list will be created by new branch, update once merged # Generate a sorted underlying vote distribution sorted_tally = sorted(self.reported.tally.items(), key=lambda x: x[1], reverse=True) self.vote_dist = [(sorted_tally[0][0], sorted_tally[0][1])] current = sorted_tally[0][1] for i in range(1, len(sorted_tally)): current += sorted_tally[i][1] self.vote_dist.append((sorted_tally[i][0], current)) self.vote_dist.append(('invalid', self.contest_ballots)) def trial(self, seed): """Execute a 1-round athena audit (using r2b2.athena.Athena)""" r.seed(seed) # Draw a sample of a given size sample = [0 for i in range(len(self.vote_dist))] for i in range(self.sample_size): ballot = r.randint(1, self.contest_ballots) for j in range(len(sample)): if ballot <= self.vote_dist[j][1]: sample[j] += 1 break relevant_sample_size = self.sample_size - sample[-1] # Perform audit computations self.audit._reset() self.audit.rounds.append(relevant_sample_size) self.audit.current_dist_null() self.audit.current_dist_reported() point_null = self.audit.distribution_null[sample[0]] point_reported = self.audit.distribution_reported_tally[sample[0]] p_value = self.audit.compute_risk(sample[0], relevant_sample_size) if p_value <= self.alpha and self.delta * point_reported > point_null: stop = True else: stop = False return { 'stop': stop, 'p_value': p_value, 'delta_computed': (point_null / point_reported), 'sample_size': self.sample_size, 'relevant_sample_size': relevant_sample_size, 'winner_ballots': sample[0] } def analyze(self, verbose: bool = False, hist: bool = False): """Analyse trials to get experimental stopping probability""" if self.db_mode: trials = self.db.trial_lookup(self.sim_id) else: trials = self.trials num_trials = 0 stopped = 0 winner_ballot_dist = [] risk_dist = [] delta_dist = [] for trial in trials: num_trials += 1 if trial['stop']: stopped += 1 winner_ballot_dist.append(trial['winner_ballots']) risk_dist.append(trial['p_value']) delta_dist.append(trial['delta_computed']) # TODO: insert verbose and histograms # Update simulation entry to include analysis if self.db_mode: self.db.update_analysis(self.sim_id, (stopped / num_trials)) return stopped / num_trials
def test_exceptions(): contest = Contest(100000, { 'A': 60000, 'B': 40000 }, 1, ['A'], ContestType.MAJORITY) with pytest.raises(ValueError): Athena(.1, 0, .1, contest) athena = Athena(.1, 1, .1, contest) with pytest.raises(Exception): athena.stopping_condition_pairwise('A-B') athena.rounds.append(10) with pytest.raises(ValueError): athena.stopping_condition_pairwise('X') athena.rounds = [] with pytest.raises(ValueError): athena.compute_min_winner_ballots(athena.sub_audits['A-B'], []) with pytest.raises(ValueError): athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [0]) with pytest.raises(ValueError): athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [1, 2]) with pytest.raises(ValueError): athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20, 20]) with pytest.raises(ValueError): athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20, 19]) with pytest.raises(ValueError): athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [10001]) athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20]) with pytest.raises(ValueError): athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20]) with pytest.raises(ValueError): athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [19]) with pytest.raises(ValueError): athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [10001]) contest2 = Contest(100, {'A': 60, 'B': 30}, 1, ['A'], ContestType.MAJORITY) athena2 = Athena(0.1, 1, 1.0, contest2) with pytest.raises(ValueError): athena2.compute_min_winner_ballots(athena2.sub_audits['A-B'], [91]) athena2.rounds.append(10) with pytest.raises(Exception): athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B']) athena2.rounds = [] with pytest.raises(ValueError): athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B'], 0) with pytest.raises(ValueError): athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B'], 200) with pytest.raises(ValueError): athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B'], 0)
def test_athena_next_sample_size(): # TODO: Create tests for ahtena next sample size simple_athena = Athena(0.1, 1, 0.1, default_contest) simple_athena.next_sample_size() pass