def test_bulk_athena():
# Same as Minerva (that is, delta = infinity)
# Ballot-by-ballot Minerva should yield identical stopping rules to BRAVO.
contest = Contest(100000, {
'A': 60000,
'B': 40000
}, 1, ['A'], ContestType.MAJORITY)
athena = Athena(.1, 2**31 - 1, .01, contest)
athena.compute_all_min_winner_ballots(athena.sub_audits['A-B'])
# p0 not hardcoded as .5 for scalability with odd total contest ballots.
p0 = (athena.contest.contest_ballots // 2) / athena.contest.contest_ballots
log_winner_multiplier = math.log(
athena.sub_audits['A-B'].sub_contest.winner_prop / p0)
log_loser_multiplier = math.log(
(1 - athena.sub_audits['A-B'].sub_contest.winner_prop) / p0)
log_rhs = math.log(1 / athena.alpha)
for i in range(len(athena.rounds)):
n = athena.rounds[i]
kmin = athena.sub_audits['A-B'].min_winner_ballots[i]
# Assert this kmin satisfies ratio, but a kmin one less does not.
assert kmin * log_winner_multiplier + (
n - kmin) * log_loser_multiplier > log_rhs
assert (kmin - 1) * log_winner_multiplier + (
n - kmin + 1) * log_loser_multiplier <= log_rhs
def test_simple_athena():
simple_athena = Athena(.1, 2**31 - 1, .1, default_contest)
assert simple_athena.alpha == .1
assert simple_athena.beta == 0.0
assert simple_athena.delta == 2**31 - 1
assert simple_athena.max_fraction_to_draw == .1
assert len(simple_athena.rounds) == 0
assert len(simple_athena.sub_audits['a-b'].min_winner_ballots) == 0
assert simple_athena.get_risk_level() is None
def test_athena_minerva_paper():
contest = Contest(100000, {
'A': 75000,
'B': 25000
}, 1, ['A'], ContestType.MAJORITY)
athena = Athena(.1, 1, .1, contest)
minerva = Minerva(.1, .1, contest)
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [50])
minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [50])
# From Athena paper
assert athena.sub_audits['A-B'].min_winner_ballots == [32]
assert minerva.sub_audits['A-B'].min_winner_ballots == [31]
def input_audit(contest: Contest,
alpha: float = None,
max_fraction_to_draw: float = None,
audit_type: str = None,
delta: float = None) -> Audit:
# Create an audit from user-input.
click.echo('\nCreate a new Audit')
click.echo('==================\n')
if alpha is None:
alpha = click.prompt('Enter the desired risk limit', type=click.FloatRange(0.0, 1.0))
if max_fraction_to_draw is None:
max_fraction_to_draw = click.prompt('Enter the maximum fraction of contest ballots to draw', type=click.FloatRange(0.0, 1.0))
if audit_type is None:
audit_type = click.prompt('Select an audit type', type=audit_types)
if delta is None and audit_type == 'athena':
delta = click.prompt('Enter the Athena delta value', type=click.FloatRange(0.0))
if audit_type == 'brla':
return BRLA(alpha, max_fraction_to_draw, contest)
elif audit_type == 'minerva':
return Minerva(alpha, max_fraction_to_draw, contest)
elif audit_type == 'athena':
return Athena(alpha, delta, max_fraction_to_draw, contest)
# TODO: add creation for other types of audits.
return None
def test_athena_execute_round():
contest = Contest(100000, {
'A': 75000,
'B': 25000
}, 1, ['A'], ContestType.MAJORITY)
athena = Athena(.1, 1, .1, contest)
assert not athena.execute_round(50, {'A': 31, 'B': 19})
assert not athena.stopped
assert athena.sample_ballots['A'] == [31]
assert athena.sample_ballots['B'] == [19]
assert not athena.sub_audits['A-B'].stopped
assert athena.rounds == [50]
assert athena.execute_round(100, {'A': 70, 'B': 30})
assert athena.stopped
assert athena.sample_ballots['A'] == [31, 70]
assert athena.sample_ballots['B'] == [19, 30]
assert athena.sub_audits['A-B'].stopped
assert athena.rounds == [50, 100]
assert athena.get_risk_level() < 0.1
def __init__(self,
alpha,
delta,
reported,
sample_size,
db_mode=True,
db_host='localhost',
db_name='r2b2',
db_port=27017,
user='******',
pwd='icanwrite',
*args,
**kwargs):
super().__init__('athena', alpha, reported, 'tie', True, db_mode,
db_host, db_port, db_name, user, pwd, *args, **kwargs)
self.sample_size = sample_size
self.total_relevant_ballots = sum(self.reported.tally.values())
# FIXME: temporary until pairwise contest fix is implemented
self.contest_ballots = self.reported.contest_ballots
self.reported.contest_ballots = self.total_relevant_ballots
self.reported.winner_prop = self.reported.tally[
self.reported.reported_winners[0]] / self.reported.contest_ballots
self.delta = delta
self.audit = Athena(self.alpha, self.delta, 1.0, self.reported)
if sample_size < self.audit.min_sample_size:
raise ValueError(
'Sample size is less than minimum sample size for audit.')
# FIXME: sorted candidate list will be created by new branch, update once merged
# Generate a sorted underlying vote distribution
sorted_tally = sorted(self.reported.tally.items(),
key=lambda x: x[1],
reverse=True)
self.vote_dist = [(sorted_tally[0][0],
self.total_relevant_ballots // 2)]
for i in range(1, len(sorted_tally)):
self.vote_dist.append(
(sorted_tally[i][0], self.total_relevant_ballots))
self.vote_dist.append(('invalid', self.contest_ballots))
class AthenaOneRoundRisk(Simulation):
"""Simulate a 1-round Athena audit for a given sample size to compute risk limit."""
delta: float
sample_size: int
total_relevant_ballots: int
vote_dist: List[Tuple[str, int]]
audit: Athena
def __init__(self,
alpha,
delta,
reported,
sample_size,
db_mode=True,
db_host='localhost',
db_name='r2b2',
db_port=27017,
user='******',
pwd='icanwrite',
*args,
**kwargs):
super().__init__('athena', alpha, reported, 'tie', True, db_mode,
db_host, db_port, db_name, user, pwd, *args, **kwargs)
self.sample_size = sample_size
self.total_relevant_ballots = sum(self.reported.tally.values())
# FIXME: temporary until pairwise contest fix is implemented
self.contest_ballots = self.reported.contest_ballots
self.reported.contest_ballots = self.total_relevant_ballots
self.reported.winner_prop = self.reported.tally[
self.reported.reported_winners[0]] / self.reported.contest_ballots
self.delta = delta
self.audit = Athena(self.alpha, self.delta, 1.0, self.reported)
if sample_size < self.audit.min_sample_size:
raise ValueError(
'Sample size is less than minimum sample size for audit.')
# FIXME: sorted candidate list will be created by new branch, update once merged
# Generate a sorted underlying vote distribution
sorted_tally = sorted(self.reported.tally.items(),
key=lambda x: x[1],
reverse=True)
self.vote_dist = [(sorted_tally[0][0],
self.total_relevant_ballots // 2)]
for i in range(1, len(sorted_tally)):
self.vote_dist.append(
(sorted_tally[i][0], self.total_relevant_ballots))
self.vote_dist.append(('invalid', self.contest_ballots))
def trial(self, seed):
"""Execute a 1-round athena audit (using r2b2.athena.Athena)"""
r.seed(seed)
# Draw a sample of a given size
sample = [0 for i in range(len(self.vote_dist))]
for i in range(self.sample_size):
ballot = r.randint(1, self.contest_ballots)
for j in range(len(sample)):
if ballot <= self.vote_dist[j][1]:
sample[j] += 1
break
relevant_sample_size = self.sample_size - sample[-1]
# Perform audit computations
self.audit._reset()
self.audit.rounds.append(relevant_sample_size)
self.audit.current_dist_null()
self.audit.current_dist_reported()
point_null = self.audit.distribution_null[sample[0]]
point_reported = self.audit.distribution_reported_tally[sample[0]]
p_value = self.audit.compute_risk(sample[0], relevant_sample_size)
if p_value <= self.alpha and self.delta * point_reported > point_null:
stop = True
else:
stop = False
return {
'stop': stop,
'p_value': p_value,
'delta_computed': (point_null / point_reported),
'sample_size': self.sample_size,
'relevant_sample_size': relevant_sample_size,
'winner_ballots': sample[0]
}
def analyze(self, verbose: bool = False, hist: bool = False):
"""Analyze trials to get experimental risk.
Args:
verbose (bool): If true, analyze will print simulation analysis information.
hist (bool): If true, analyze will generate and display 2 histograms: winner
ballots found in the sample size and computed risk.
"""
if self.db_mode:
trials = self.db.trial_lookup(self.sim_id)
else:
trials = self.trials
num_trials = 0
stopped = 0
total_risk = 0
total_delta = 0
total_relevant_sampled = 0
winner_ballot_dist = []
risk_dist = []
delta_dist = []
for trial in trials:
num_trials += 1
if trial['stop']:
stopped += 1
total_relevant_sampled += trial['relevant_sample_size']
winner_ballot_dist.append(trial['winner_ballots'])
total_risk += trial['p_value']
risk_dist.append(trial['p_value'])
total_delta += trial['delta_computed']
delta_dist.append(trial['delta_computed'])
if verbose:
print('Analysis\n========')
print('Underlying election is tied\n')
print('Number of trials: {}'.format(num_trials))
print('Number of stopped: {}'.format(stopped))
print('Risk Limit: {:%}'.format(self.alpha))
print('Risk Computed: {:%}'.format(stopped / num_trials))
print('Delta Condition: {}'.format(self.delta))
print('Avg. Delta Computed: {}'.format(total_delta / num_trials))
if hist:
histogram(
winner_ballot_dist,
'Winner ballots found in sample of size: {}'.format(
self.sample_size))
histogram(risk_dist, 'Risk (p_value) dist.')
histogram(delta_dist, 'Delta (computed) dist.')
# Update simulation entry to include analysis
if self.db_mode:
self.db.update_analysis(self.sim_id, (stopped / num_trials))
return stopped / num_trials
class AthenaOneRoundStoppingProb(Simulation):
"""Simulate a 1-round Athena audit for a given sample size to compute stopping probability."""
delta: float
sample_size: int
total_relevant_ballots: int
vote_dist: List[Tuple[str, int]]
audit: Athena
def __init__(self,
alpha,
delta,
reported,
sample_size,
db_mode=True,
db_host='localhost',
db_name='r2b2',
db_port=27017,
user='******',
pwd='icanwrite',
*args,
**kwargs):
super().__init__('athena', alpha, reported, 'reported', True, db_mode,
db_host, db_port, db_name, user, pwd, *args, **kwargs)
self.delta = delta
self.sample_size = sample_size
self.total_relevant_ballots = sum(self.reported.tally.values())
# FIXME: temporary until pairwise contest fix is implemented
self.contest_ballots = self.reported.contest_ballots
self.reported.contest_ballots = self.total_relevant_ballots
self.reported.winner_prop = self.reported.tally[
self.reported.reported_winners[0]] / self.reported.contest_ballots
self.audit = Athena(self.alpha, self.delta, 1.0, self.reported)
if sample_size < self.audit.min_sample_size:
raise ValueError(
'Sample size is less than minimum sample size for audit')
# FIXME: sorted candidate list will be created by new branch, update once merged
# Generate a sorted underlying vote distribution
sorted_tally = sorted(self.reported.tally.items(),
key=lambda x: x[1],
reverse=True)
self.vote_dist = [(sorted_tally[0][0], sorted_tally[0][1])]
current = sorted_tally[0][1]
for i in range(1, len(sorted_tally)):
current += sorted_tally[i][1]
self.vote_dist.append((sorted_tally[i][0], current))
self.vote_dist.append(('invalid', self.contest_ballots))
def trial(self, seed):
"""Execute a 1-round athena audit (using r2b2.athena.Athena)"""
r.seed(seed)
# Draw a sample of a given size
sample = [0 for i in range(len(self.vote_dist))]
for i in range(self.sample_size):
ballot = r.randint(1, self.contest_ballots)
for j in range(len(sample)):
if ballot <= self.vote_dist[j][1]:
sample[j] += 1
break
relevant_sample_size = self.sample_size - sample[-1]
# Perform audit computations
self.audit._reset()
self.audit.rounds.append(relevant_sample_size)
self.audit.current_dist_null()
self.audit.current_dist_reported()
point_null = self.audit.distribution_null[sample[0]]
point_reported = self.audit.distribution_reported_tally[sample[0]]
p_value = self.audit.compute_risk(sample[0], relevant_sample_size)
if p_value <= self.alpha and self.delta * point_reported > point_null:
stop = True
else:
stop = False
return {
'stop': stop,
'p_value': p_value,
'delta_computed': (point_null / point_reported),
'sample_size': self.sample_size,
'relevant_sample_size': relevant_sample_size,
'winner_ballots': sample[0]
}
def analyze(self, verbose: bool = False, hist: bool = False):
"""Analyse trials to get experimental stopping probability"""
if self.db_mode:
trials = self.db.trial_lookup(self.sim_id)
else:
trials = self.trials
num_trials = 0
stopped = 0
winner_ballot_dist = []
risk_dist = []
delta_dist = []
for trial in trials:
num_trials += 1
if trial['stop']:
stopped += 1
winner_ballot_dist.append(trial['winner_ballots'])
risk_dist.append(trial['p_value'])
delta_dist.append(trial['delta_computed'])
# TODO: insert verbose and histograms
# Update simulation entry to include analysis
if self.db_mode:
self.db.update_analysis(self.sim_id, (stopped / num_trials))
return stopped / num_trials
def test_exceptions():
contest = Contest(100000, {
'A': 60000,
'B': 40000
}, 1, ['A'], ContestType.MAJORITY)
with pytest.raises(ValueError):
Athena(.1, 0, .1, contest)
athena = Athena(.1, 1, .1, contest)
with pytest.raises(Exception):
athena.stopping_condition_pairwise('A-B')
athena.rounds.append(10)
with pytest.raises(ValueError):
athena.stopping_condition_pairwise('X')
athena.rounds = []
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [0])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [1, 2])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20, 20])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20, 19])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [10001])
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [19])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [10001])
contest2 = Contest(100, {'A': 60, 'B': 30}, 1, ['A'], ContestType.MAJORITY)
athena2 = Athena(0.1, 1, 1.0, contest2)
with pytest.raises(ValueError):
athena2.compute_min_winner_ballots(athena2.sub_audits['A-B'], [91])
athena2.rounds.append(10)
with pytest.raises(Exception):
athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B'])
athena2.rounds = []
with pytest.raises(ValueError):
athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B'], 0)
with pytest.raises(ValueError):
athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B'], 200)
with pytest.raises(ValueError):
athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B'], 0)
def test_athena_next_sample_size():
# TODO: Create tests for ahtena next sample size
simple_athena = Athena(0.1, 1, 0.1, default_contest)
simple_athena.next_sample_size()
pass