def test_exceptions():
contest = Contest(100000, {
'A': 60000,
'B': 40000
}, 1, ['A'], ContestType.MAJORITY)
with pytest.raises(ValueError):
Athena(.1, 0, .1, contest)
athena = Athena(.1, 1, .1, contest)
with pytest.raises(Exception):
athena.stopping_condition_pairwise('A-B')
athena.rounds.append(10)
with pytest.raises(ValueError):
athena.stopping_condition_pairwise('X')
athena.rounds = []
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [0])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [1, 2])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20, 20])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20, 19])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [10001])
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [20])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [19])
with pytest.raises(ValueError):
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [10001])
contest2 = Contest(100, {'A': 60, 'B': 30}, 1, ['A'], ContestType.MAJORITY)
athena2 = Athena(0.1, 1, 1.0, contest2)
with pytest.raises(ValueError):
athena2.compute_min_winner_ballots(athena2.sub_audits['A-B'], [91])
athena2.rounds.append(10)
with pytest.raises(Exception):
athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B'])
athena2.rounds = []
with pytest.raises(ValueError):
athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B'], 0)
with pytest.raises(ValueError):
athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B'], 200)
with pytest.raises(ValueError):
athena2.compute_all_min_winner_ballots(athena2.sub_audits['A-B'], 0)
def input_audit(contest: Contest,
alpha: float = None,
max_fraction_to_draw: float = None,
audit_type: str = None,
delta: float = None) -> Audit:
# Create an audit from user-input.
click.echo('\nCreate a new Audit')
click.echo('==================\n')
if alpha is None:
alpha = click.prompt('Enter the desired risk limit', type=click.FloatRange(0.0, 1.0))
if max_fraction_to_draw is None:
max_fraction_to_draw = click.prompt('Enter the maximum fraction of contest ballots to draw', type=click.FloatRange(0.0, 1.0))
if audit_type is None:
audit_type = click.prompt('Select an audit type', type=audit_types)
if delta is None and audit_type == 'athena':
delta = click.prompt('Enter the Athena delta value', type=click.FloatRange(0.0))
if audit_type == 'brla':
return BRLA(alpha, max_fraction_to_draw, contest)
elif audit_type == 'minerva':
return Minerva(alpha, max_fraction_to_draw, contest)
elif audit_type == 'athena':
return Athena(alpha, delta, max_fraction_to_draw, contest)
# TODO: add creation for other types of audits.
return None
def test_bulk_athena():
# Same as Minerva (that is, delta = infinity)
# Ballot-by-ballot Minerva should yield identical stopping rules to BRAVO.
contest = Contest(100000, {
'A': 60000,
'B': 40000
}, 1, ['A'], ContestType.MAJORITY)
athena = Athena(.1, 2**31 - 1, .01, contest)
athena.compute_all_min_winner_ballots(athena.sub_audits['A-B'])
# p0 not hardcoded as .5 for scalability with odd total contest ballots.
p0 = (athena.contest.contest_ballots // 2) / athena.contest.contest_ballots
log_winner_multiplier = math.log(
athena.sub_audits['A-B'].sub_contest.winner_prop / p0)
log_loser_multiplier = math.log(
(1 - athena.sub_audits['A-B'].sub_contest.winner_prop) / p0)
log_rhs = math.log(1 / athena.alpha)
for i in range(len(athena.rounds)):
n = athena.rounds[i]
kmin = athena.sub_audits['A-B'].min_winner_ballots[i]
# Assert this kmin satisfies ratio, but a kmin one less does not.
assert kmin * log_winner_multiplier + (
n - kmin) * log_loser_multiplier > log_rhs
assert (kmin - 1) * log_winner_multiplier + (
n - kmin + 1) * log_loser_multiplier <= log_rhs
def test_simple_athena():
simple_athena = Athena(.1, 2**31 - 1, .1, default_contest)
assert simple_athena.alpha == .1
assert simple_athena.beta == 0.0
assert simple_athena.delta == 2**31 - 1
assert simple_athena.max_fraction_to_draw == .1
assert len(simple_athena.rounds) == 0
assert len(simple_athena.sub_audits['a-b'].min_winner_ballots) == 0
assert simple_athena.get_risk_level() is None
def test_athena_minerva_paper():
contest = Contest(100000, {
'A': 75000,
'B': 25000
}, 1, ['A'], ContestType.MAJORITY)
athena = Athena(.1, 1, .1, contest)
minerva = Minerva(.1, .1, contest)
athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [50])
minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [50])
# From Athena paper
assert athena.sub_audits['A-B'].min_winner_ballots == [32]
assert minerva.sub_audits['A-B'].min_winner_ballots == [31]
def test_athena_execute_round():
contest = Contest(100000, {
'A': 75000,
'B': 25000
}, 1, ['A'], ContestType.MAJORITY)
athena = Athena(.1, 1, .1, contest)
assert not athena.execute_round(50, {'A': 31, 'B': 19})
assert not athena.stopped
assert athena.sample_ballots['A'] == [31]
assert athena.sample_ballots['B'] == [19]
assert not athena.sub_audits['A-B'].stopped
assert athena.rounds == [50]
assert athena.execute_round(100, {'A': 70, 'B': 30})
assert athena.stopped
assert athena.sample_ballots['A'] == [31, 70]
assert athena.sample_ballots['B'] == [19, 30]
assert athena.sub_audits['A-B'].stopped
assert athena.rounds == [50, 100]
assert athena.get_risk_level() < 0.1
def __init__(self,
alpha,
delta,
reported,
sample_size,
db_mode=True,
db_host='localhost',
db_name='r2b2',
db_port=27017,
user='******',
pwd='icanwrite',
*args,
**kwargs):
super().__init__('athena', alpha, reported, 'tie', True, db_mode,
db_host, db_port, db_name, user, pwd, *args, **kwargs)
self.sample_size = sample_size
self.total_relevant_ballots = sum(self.reported.tally.values())
# FIXME: temporary until pairwise contest fix is implemented
self.contest_ballots = self.reported.contest_ballots
self.reported.contest_ballots = self.total_relevant_ballots
self.reported.winner_prop = self.reported.tally[
self.reported.reported_winners[0]] / self.reported.contest_ballots
self.delta = delta
self.audit = Athena(self.alpha, self.delta, 1.0, self.reported)
if sample_size < self.audit.min_sample_size:
raise ValueError(
'Sample size is less than minimum sample size for audit.')
# FIXME: sorted candidate list will be created by new branch, update once merged
# Generate a sorted underlying vote distribution
sorted_tally = sorted(self.reported.tally.items(),
key=lambda x: x[1],
reverse=True)
self.vote_dist = [(sorted_tally[0][0],
self.total_relevant_ballots // 2)]
for i in range(1, len(sorted_tally)):
self.vote_dist.append(
(sorted_tally[i][0], self.total_relevant_ballots))
self.vote_dist.append(('invalid', self.contest_ballots))
def test_athena_next_sample_size():
# TODO: Create tests for ahtena next sample size
simple_athena = Athena(0.1, 1, 0.1, default_contest)
simple_athena.next_sample_size()
pass