def test_multiple_candidate_minerva():
test_file = 'src/r2b2/tests/data/full_multi_cand.json'
with open(test_file, 'r') as tf:
data = json.load(tf)
test = 'testx'
# Get contest from test
contest_data = data[test]['election']['contests']['contest_1']
contest = Contest(contest_data['contest_ballots'],
contest_data['tally'], contest_data['num_winners'],
contest_data['reported_winners'],
ContestType[contest_data['contest_type']])
audit = Minerva(data[test]['alpha'], 1.0, contest)
for r in data[test]['rounds']:
sample_raw = data[test]['rounds'][r]['pvalue']['observations']
sample_size = sum(sample_raw)
sample = {}
for i, c in enumerate(contest.candidates):
sample[c] = sample_raw[i]
audit.execute_round(sample_size, sample)
assert abs(
data[test]['rounds'][r]['pvalue']['expected']['pvalue'] -
audit.pvalue_schedule[-1]) < tol
for pair in data[test]['rounds'][r]['pvalue']['expected'][
'pairwise']:
assert abs(data[test]['rounds'][r]['pvalue']['expected']
['pairwise'][pair] -
audit.sub_audits[pair].pvalue_schedule[-1]) < tol
def test_execute_round_minerva():
contest = Contest(100000, {
'A': 60000,
'B': 40000
}, 1, ['A'], ContestType.MAJORITY)
minerva = Minerva(.1, .1, contest)
assert not minerva.execute_round(100, {'A': 57, 'B': 43})
assert not minerva.stopped
assert minerva.sample_ballots['A'] == [57]
assert minerva.sample_ballots['B'] == [43]
assert not minerva.sub_audits['A-B'].stopped
assert minerva.rounds == [100]
assert not minerva.execute_round(200, {'A': 112, 'B': 88})
assert not minerva.stopped
assert minerva.sample_ballots['A'] == [57, 112]
assert minerva.sample_ballots['B'] == [43, 88]
assert not minerva.sub_audits['A-B'].stopped
assert minerva.rounds == [100, 200]
assert minerva.execute_round(400, {'A': 221, 'B': 179})
assert minerva.stopped
assert minerva.sample_ballots['A'] == [57, 112, 221]
assert minerva.sample_ballots['B'] == [43, 88, 179]
assert minerva.sub_audits['A-B'].stopped
assert minerva.rounds == [100, 200, 400]
assert minerva.get_risk_level() < 0.1
def test_minerva_arlo():
with open('tests/data/arlo_tests.json', 'r') as tf:
data = json.load(tf)
for test in data:
contest_data = data[test]['contest']
contest = Contest(contest_data['contest_ballots'], contest_data['tally'], contest_data['num_winners'],
contest_data['reported_winners'], ContestType[contest_data['contest_type']])
if data[test]['audit_type'] != 'minerva':
pass
audit = Minerva(data[test]['alpha'], 1.0, contest)
for r in data[test]['rounds']:
round_data = data[test]['rounds'][r]
audit.execute_round(round_data['sample_size'], round_data['sample'])
assert audit.stopped == bool(data[test]['expected']['stopped'])
assert abs(audit.get_risk_level() - data[test]['expected']['pvalue']) < tol
def test_minerva_georgia_senate_2020():
ga_senate_race = Contest(2453876 + 2358432, {
'A': 2453876,
'B': 2358432
}, 1, ['A'], ContestType.PLURALITY)
ga_senate_audit = Minerva(.1, 1.0, ga_senate_race)
irrelevant_scale_up = 1.0238785631
estimates = []
for sprob in [.7, .8, .9]:
estimates.append(
math.ceil(irrelevant_scale_up *
ga_senate_audit.next_sample_size(sprob=sprob)))
assert estimates == [10486, 13205, 18005]
ga_senate_audit.execute_round(9903, {'A': 4950, 'B': 9903 - 4950})
assert abs(ga_senate_audit.pvalue_schedule[-1] -
0.527638189598802) < .000001
ga_senate_audit.execute_round(24000, {'A': 11900, 'B': 24000 - 11900})
assert abs(ga_senate_audit.pvalue_schedule[-1] -
2.663358309286826) < .000001
ga_senate_audit.execute_round(45600, {'A': 24000, 'B': 45600 - 24000})
assert abs(ga_senate_audit.pvalue_schedule[-1]) < 0.000001
ga_senate_audit = Minerva(.1, 1.0, ga_senate_race)
ga_senate_audit.execute_round(17605, {'A': 8900, 'B': 17605 - 8900})
assert abs(ga_senate_audit.get_risk_level() - 0.081750333563781) < .000001
ga_senate_audit = Minerva(.1, 1.0, ga_senate_race)
ga_senate_audit.execute_round(17605, {'A': 17605, 'B': 0})
assert ga_senate_audit.get_risk_level() == 0
ga_senate_audit = Minerva(.1, 1.0, ga_senate_race)
ga_senate_audit.execute_round(17605, {'A': 0, 'B': 17605})
assert abs(ga_senate_audit.get_risk_level() - 1) < 0.000001
def test_minerva_second_round_estimate_2016():
with open('tests/data/2016_pres_trials.json', 'r') as json_file:
data = json.load(json_file)
out = {}
out['data_check'] = {}
for state in data:
out['data_check'][state] = {}
clinton = data[state]['tally']['Clinton']
trump = data[state]['tally']['Trump']
print(state)
tally = {"Clinton": clinton, "Trump": trump}
margin = abs((clinton - trump) / (clinton + trump))
if margin < .10:
continue
contest = Contest(clinton + trump, tally, 1,
[max(tally, key=tally.get)], ContestType.PLURALITY)
if tally['Clinton'] > tally['Trump']:
rep_winner = 'Clinton'
rep_loser = 'Trump'
else:
rep_winner = 'Trump'
rep_loser = 'Clinton'
for sim_type in [
'underlying_reported_first_5',
'underlying_reported_not_stop_5', 'underlying_tied_first_5'
]:
out['data_check'][state][sim_type] = []
for trial in data[state][sim_type]:
n = trial['relevant_sample_size']
k = trial['winner_ballots']
minerva = Minerva(.1, 1.0, contest)
minerva.execute_round(n, {rep_winner: k, rep_loser: n - k})
p_value = minerva.get_risk_level()
stop = minerva.stopped
if stop:
minerva.next_min_winner_ballots()
minerva.truncate_dist_null()
minerva.truncate_dist_reported()
next_round_data = minerva.next_sample_size(verbose=True)
out['data_check'][state][sim_type].append({
"n":
n,
"k":
k,
"p_value":
p_value,
"stop":
bool(stop),
"kmin":
minerva.sub_audits[rep_winner + '-' +
rep_loser].min_winner_ballots[-1],
"next_round_size":
next_round_data[0],
"next_round_kmin":
next_round_data[1],
"next_round_sprob":
next_round_data[2]
})
with open(
'tests/data/test_minerva_second_round_estimate_2016.json',
'w') as output:
json.dump(out, output, sort_keys=True, indent=4)
# Now that the file has been generated, compare to PV version.
with open('tests/data/gm_test_minerva_second_round_estimate_2016.json',
'r') as json_file:
data_canonical = json.load(json_file)
with open('tests/data/test_minerva_second_round_estimate_2016.json',
'r') as json_file:
data_test = json.load(json_file)
for state in data_canonical['data_check']:
if data_canonical['data_check'][state] == {}:
continue
for sim_type in [
'underlying_reported_first_5',
'underlying_reported_not_stop_5', 'underlying_tied_first_5'
]:
for i in range(5):
assert data_canonical['data_check'][state][sim_type][i][
'n'] == data_test['data_check'][state][sim_type][i]['n']
assert data_canonical['data_check'][state][sim_type][i][
'k'] == data_test['data_check'][state][sim_type][i]['k']
assert data_canonical['data_check'][state][sim_type][i][
'kmin'] == data_test['data_check'][state][sim_type][i][
'kmin']
assert data_canonical['data_check'][state][sim_type][i][
'stop'] == data_test['data_check'][state][sim_type][i][
'stop']
assert data_canonical['data_check'][state][sim_type][i]['next_round_size'] == \
data_test['data_check'][state][sim_type][i]['next_round_size']
assert abs(data_canonical['data_check'][state][sim_type][i]
['p_value'] - data_test['data_check'][state]
[sim_type][i]['p_value']) < .000001
assert abs(data_canonical['data_check'][state][sim_type][i]
['next_round_sprob'] - data_test['data_check']
[state][sim_type][i]['next_round_sprob']) < .000001