Beispiel #1
0
def test_bulk_minerva():
    # Ballot-by-ballot Minerva should yield identical stopping rules to BRAVO.
    contest = Contest(100000, {
        'A': 60000,
        'B': 40000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva = Minerva(.1, .01, contest)
    minerva.compute_all_min_winner_ballots(minerva.sub_audits['A-B'])
    # p0 not hardcoded as .5 for scalability with odd total contest ballots.
    p0 = (minerva.contest.contest_ballots //
          2) / minerva.contest.contest_ballots
    log_winner_multiplier = math.log(
        minerva.sub_audits['A-B'].sub_contest.winner_prop / p0)
    log_loser_multiplier = math.log(
        (1 - minerva.sub_audits['A-B'].sub_contest.winner_prop) / p0)
    log_rhs = math.log(1 / minerva.alpha)

    for i in range(len(minerva.rounds)):
        n = minerva.rounds[i]
        kmin = minerva.sub_audits['A-B'].min_winner_ballots[i]
        # Assert this kmin satisfies ratio, but a kmin one less does not.
        assert kmin * log_winner_multiplier + (
            n - kmin) * log_loser_multiplier > log_rhs
        assert (kmin - 1) * log_winner_multiplier + (
            n - kmin + 1) * log_loser_multiplier <= log_rhs
Beispiel #2
0
def test_multiple_candidate_minerva():
    test_file = 'src/r2b2/tests/data/full_multi_cand.json'
    with open(test_file, 'r') as tf:
        data = json.load(tf)
        test = 'testx'
        # Get contest from test
        contest_data = data[test]['election']['contests']['contest_1']
        contest = Contest(contest_data['contest_ballots'],
                          contest_data['tally'], contest_data['num_winners'],
                          contest_data['reported_winners'],
                          ContestType[contest_data['contest_type']])
        audit = Minerva(data[test]['alpha'], 1.0, contest)

        for r in data[test]['rounds']:
            sample_raw = data[test]['rounds'][r]['pvalue']['observations']
            sample_size = sum(sample_raw)
            sample = {}
            for i, c in enumerate(contest.candidates):
                sample[c] = sample_raw[i]
            audit.execute_round(sample_size, sample)
            assert abs(
                data[test]['rounds'][r]['pvalue']['expected']['pvalue'] -
                audit.pvalue_schedule[-1]) < tol
            for pair in data[test]['rounds'][r]['pvalue']['expected'][
                    'pairwise']:
                assert abs(data[test]['rounds'][r]['pvalue']['expected']
                           ['pairwise'][pair] -
                           audit.sub_audits[pair].pvalue_schedule[-1]) < tol
Beispiel #3
0
def test_sentinel():
    contest = Contest(100000, {
        'A': 60000,
        'B': 40000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva = Minerva(.1, .1, contest)
    minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [13, 14, 15])
    assert minerva.sub_audits['A-B'].min_winner_ballots == [13, None, 14]
Beispiel #4
0
def test_minerva_first_round_gaussian_estimate():
    contest4 = Contest(1000000, {
        'A': 502000,
        'B': 498000
    }, 1, ['A'], ContestType.PLURALITY)
    minerva4 = Minerva(.1, 1.0, contest4)

    assert minerva4.next_sample_size_gaussian() == 429778
Beispiel #5
0
def test_minerva_kmins():
    contest = Contest(100000, {
        'A': 60000,
        'B': 40000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva = Minerva(.1, .1, contest)
    minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'],
                                       [100, 200, 400])

    # From existing software
    assert minerva.sub_audits['A-B'].min_winner_ballots == [58, 113, 221]
def test_athena_minerva_paper():
    contest = Contest(100000, {
        'A': 75000,
        'B': 25000
    }, 1, ['A'], ContestType.MAJORITY)
    athena = Athena(.1, 1, .1, contest)
    minerva = Minerva(.1, .1, contest)
    athena.compute_min_winner_ballots(athena.sub_audits['A-B'], [50])
    minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [50])

    # From Athena paper
    assert athena.sub_audits['A-B'].min_winner_ballots == [32]
    assert minerva.sub_audits['A-B'].min_winner_ballots == [31]
Beispiel #7
0
def test_min_sample_size():
    contest1 = Contest(100000, {
        'A': 60000,
        'B': 40000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva1 = Minerva(.1, .1, contest1)
    contest2 = Contest(100000, {
        'A': 51000,
        'B': 49000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva2 = Minerva(.05, .05, contest2)

    assert minerva1.sub_audits['A-B'].min_sample_size == 13
    assert minerva2.sub_audits['A-B'].min_sample_size == 840
def test_minerva_arlo():
    with open('tests/data/arlo_tests.json', 'r') as tf:
        data = json.load(tf)
        for test in data:
            contest_data = data[test]['contest']
            contest = Contest(contest_data['contest_ballots'], contest_data['tally'], contest_data['num_winners'],
                              contest_data['reported_winners'], ContestType[contest_data['contest_type']])
            if data[test]['audit_type'] != 'minerva':
                pass
            audit = Minerva(data[test]['alpha'], 1.0, contest)
            for r in data[test]['rounds']:
                round_data = data[test]['rounds'][r]
                audit.execute_round(round_data['sample_size'], round_data['sample'])

            assert audit.stopped == bool(data[test]['expected']['stopped'])
            assert abs(audit.get_risk_level() - data[test]['expected']['pvalue']) < tol
Beispiel #9
0
def input_audit(contest: Contest,
                alpha: float = None,
                max_fraction_to_draw: float = None,
                audit_type: str = None,
                delta: float = None) -> Audit:
    # Create an audit from user-input.
    click.echo('\nCreate a new Audit')
    click.echo('==================\n')

    if alpha is None:
        alpha = click.prompt('Enter the desired risk limit', type=click.FloatRange(0.0, 1.0))
    if max_fraction_to_draw is None:
        max_fraction_to_draw = click.prompt('Enter the maximum fraction of contest ballots to draw', type=click.FloatRange(0.0, 1.0))
    if audit_type is None:
        audit_type = click.prompt('Select an audit type', type=audit_types)
    if delta is None and audit_type == 'athena':
        delta = click.prompt('Enter the Athena delta value', type=click.FloatRange(0.0))

    if audit_type == 'brla':
        return BRLA(alpha, max_fraction_to_draw, contest)
    elif audit_type == 'minerva':
        return Minerva(alpha, max_fraction_to_draw, contest)
    elif audit_type == 'athena':
        return Athena(alpha, delta, max_fraction_to_draw, contest)
    # TODO: add creation for other types of audits.
    return None
Beispiel #10
0
def test_minerva_second_round_estimate():
    contest1 = Contest(100000, {
        'A': 60000,
        'B': 40000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva1 = Minerva(.1, .1, contest1)
    minerva1.compute_min_winner_ballots(minerva1.sub_audits['A-B'], [100])
    minerva1.sample_ballots['A'].append(54)
    minerva1.sample_ballots['B'].append(100 - 54)
    contest2 = Contest(4504975 + 4617886, {
        'Trump': 4617886,
        'Clinton': 4504975
    }, 1, ['Trump'], ContestType.PLURALITY)
    minerva2 = Minerva(.1, 1.0, contest2)
    minerva2.compute_min_winner_ballots(minerva2.sub_audits['Trump-Clinton'],
                                        [45081])
    minerva2.sample_ballots['Trump'].append(22634)
    minerva2.sample_ballots['Clinton'].append(45081 - 22634)

    assert minerva1.next_sample_size() == 305
    assert minerva2.next_sample_size() == 111257
Beispiel #11
0
def test_execute_round_minerva():
    contest = Contest(100000, {
        'A': 60000,
        'B': 40000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva = Minerva(.1, .1, contest)
    assert not minerva.execute_round(100, {'A': 57, 'B': 43})
    assert not minerva.stopped
    assert minerva.sample_ballots['A'] == [57]
    assert minerva.sample_ballots['B'] == [43]
    assert not minerva.sub_audits['A-B'].stopped
    assert minerva.rounds == [100]
    assert not minerva.execute_round(200, {'A': 112, 'B': 88})
    assert not minerva.stopped
    assert minerva.sample_ballots['A'] == [57, 112]
    assert minerva.sample_ballots['B'] == [43, 88]
    assert not minerva.sub_audits['A-B'].stopped
    assert minerva.rounds == [100, 200]
    assert minerva.execute_round(400, {'A': 221, 'B': 179})
    assert minerva.stopped
    assert minerva.sample_ballots['A'] == [57, 112, 221]
    assert minerva.sample_ballots['B'] == [43, 88, 179]
    assert minerva.sub_audits['A-B'].stopped
    assert minerva.rounds == [100, 200, 400]
    assert minerva.get_risk_level() < 0.1
Beispiel #12
0
def test_minerva_first_round_estimate():

    contest1 = Contest(100000, {
        'A': 60000,
        'B': 40000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva1 = Minerva(.1, .1, contest1)
    contest2 = Contest(100000, {
        'A': 51000,
        'B': 49000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva2 = Minerva(.1, .1, contest2)
    contest3 = Contest(10000000, {
        'A': 5040799,
        'B': 10000000 - 5040799
    }, 1, ['A'], ContestType.MAJORITY)
    minerva3 = Minerva(.1, 1.0, contest3)

    assert minerva1.next_sample_size() == 179
    assert minerva2.next_sample_size() == 17272
    assert minerva3.next_sample_size() == 103483
Beispiel #13
0
def test_simple_minerva():
    simple_minerva = Minerva(.1, .1, default_contest)
    assert simple_minerva.alpha == .1
    assert simple_minerva.beta == 0.0
    assert simple_minerva.max_fraction_to_draw == .1
    assert len(simple_minerva.rounds) == 0
    assert len(simple_minerva.sub_audits) == 1
    assert simple_minerva.get_risk_level() is None
    simple_minerva.rounds.append(10)
    simple_minerva.stopped = True
    assert simple_minerva.next_sample_size() == 10
    assert simple_minerva.next_sample_size(verbose=True) == (10, 0, 1)
Beispiel #14
0
def test_kmin_upper_bound():
    contest1 = Contest(100000, {
        'A': 60000,
        'B': 40000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva1 = Minerva(.1, .1, contest1)
    contest2 = Contest(100000, {
        'A': 90000,
        'B': 10000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva2 = Minerva(.1, .1, contest2)

    assert minerva1.kmin_search_upper_bound(200,
                                            minerva1.sub_audits['A-B']) == 116
    assert minerva2.kmin_search_upper_bound(2000,
                                            minerva2.sub_audits['A-B']) == 1467
Beispiel #15
0
def test_exceptions():
    contest = Contest(100000, {
        'A': 60000,
        'B': 40000
    }, 1, ['A'], ContestType.MAJORITY)
    minerva = Minerva(.1, .1, contest)
    with pytest.raises(ValueError):
        minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [])
    with pytest.raises(ValueError):
        minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [0])
    with pytest.raises(ValueError):
        minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [1, 2])
    with pytest.raises(ValueError):
        minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [20, 20])
    with pytest.raises(ValueError):
        minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [20, 19])
    with pytest.raises(ValueError):
        minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [10001])

    minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [20])
    with pytest.raises(ValueError):
        minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [20])
    with pytest.raises(ValueError):
        minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [19])
    with pytest.raises(ValueError):
        minerva.compute_min_winner_ballots(minerva.sub_audits['A-B'], [10001])

    contest2 = Contest(100, {'A': 60, 'B': 30}, 1, ['A'], ContestType.MAJORITY)
    minerva2 = Minerva(.1, 1.0, contest2)
    with pytest.raises(ValueError):
        minerva2.compute_min_winner_ballots(minerva2.sub_audits['A-B'], [91])
    minerva2.rounds = [10]
    with pytest.raises(Exception):
        minerva2.compute_all_min_winner_ballots(minerva2.sub_audits['A-B'])
    minerva2.rounds = []
    with pytest.raises(Exception):
        minerva.compute_all_min_winner_ballots(minerva2.sub_audits['A-B'], 200)

    minerva = Minerva(.1, .1, contest)
    with pytest.raises(Exception):
        minerva.stopping_condition_pairwise('A-B')
    minerva.rounds.append(10)
    with pytest.raises(ValueError):
        minerva.stopping_condition_pairwise('x')
Beispiel #16
0
def test_minerva_georgia_senate_2020():
    ga_senate_race = Contest(2453876 + 2358432, {
        'A': 2453876,
        'B': 2358432
    }, 1, ['A'], ContestType.PLURALITY)

    ga_senate_audit = Minerva(.1, 1.0, ga_senate_race)
    irrelevant_scale_up = 1.0238785631
    estimates = []
    for sprob in [.7, .8, .9]:
        estimates.append(
            math.ceil(irrelevant_scale_up *
                      ga_senate_audit.next_sample_size(sprob=sprob)))
    assert estimates == [10486, 13205, 18005]
    ga_senate_audit.execute_round(9903, {'A': 4950, 'B': 9903 - 4950})
    assert abs(ga_senate_audit.pvalue_schedule[-1] -
               0.527638189598802) < .000001
    ga_senate_audit.execute_round(24000, {'A': 11900, 'B': 24000 - 11900})
    assert abs(ga_senate_audit.pvalue_schedule[-1] -
               2.663358309286826) < .000001
    ga_senate_audit.execute_round(45600, {'A': 24000, 'B': 45600 - 24000})
    assert abs(ga_senate_audit.pvalue_schedule[-1]) < 0.000001

    ga_senate_audit = Minerva(.1, 1.0, ga_senate_race)
    ga_senate_audit.execute_round(17605, {'A': 8900, 'B': 17605 - 8900})
    assert abs(ga_senate_audit.get_risk_level() - 0.081750333563781) < .000001

    ga_senate_audit = Minerva(.1, 1.0, ga_senate_race)
    ga_senate_audit.execute_round(17605, {'A': 17605, 'B': 0})
    assert ga_senate_audit.get_risk_level() == 0

    ga_senate_audit = Minerva(.1, 1.0, ga_senate_race)
    ga_senate_audit.execute_round(17605, {'A': 0, 'B': 17605})
    assert abs(ga_senate_audit.get_risk_level() - 1) < 0.000001
def test_minerva_second_round_estimate_2016():
    with open('tests/data/2016_pres_trials.json', 'r') as json_file:
        data = json.load(json_file)

    out = {}
    out['data_check'] = {}

    for state in data:
        out['data_check'][state] = {}

        clinton = data[state]['tally']['Clinton']
        trump = data[state]['tally']['Trump']
        print(state)
        tally = {"Clinton": clinton, "Trump": trump}
        margin = abs((clinton - trump) / (clinton + trump))
        if margin < .10:
            continue

        contest = Contest(clinton + trump, tally, 1,
                          [max(tally, key=tally.get)], ContestType.PLURALITY)
        if tally['Clinton'] > tally['Trump']:
            rep_winner = 'Clinton'
            rep_loser = 'Trump'
        else:
            rep_winner = 'Trump'
            rep_loser = 'Clinton'

        for sim_type in [
                'underlying_reported_first_5',
                'underlying_reported_not_stop_5', 'underlying_tied_first_5'
        ]:
            out['data_check'][state][sim_type] = []
            for trial in data[state][sim_type]:
                n = trial['relevant_sample_size']
                k = trial['winner_ballots']
                minerva = Minerva(.1, 1.0, contest)
                minerva.execute_round(n, {rep_winner: k, rep_loser: n - k})
                p_value = minerva.get_risk_level()
                stop = minerva.stopped
                if stop:
                    minerva.next_min_winner_ballots()
                    minerva.truncate_dist_null()
                    minerva.truncate_dist_reported()
                next_round_data = minerva.next_sample_size(verbose=True)
                out['data_check'][state][sim_type].append({
                    "n":
                    n,
                    "k":
                    k,
                    "p_value":
                    p_value,
                    "stop":
                    bool(stop),
                    "kmin":
                    minerva.sub_audits[rep_winner + '-' +
                                       rep_loser].min_winner_ballots[-1],
                    "next_round_size":
                    next_round_data[0],
                    "next_round_kmin":
                    next_round_data[1],
                    "next_round_sprob":
                    next_round_data[2]
                })

            with open(
                    'tests/data/test_minerva_second_round_estimate_2016.json',
                    'w') as output:
                json.dump(out, output, sort_keys=True, indent=4)

    # Now that the file has been generated, compare to PV version.
    with open('tests/data/gm_test_minerva_second_round_estimate_2016.json',
              'r') as json_file:
        data_canonical = json.load(json_file)
    with open('tests/data/test_minerva_second_round_estimate_2016.json',
              'r') as json_file:
        data_test = json.load(json_file)

    for state in data_canonical['data_check']:
        if data_canonical['data_check'][state] == {}:
            continue
        for sim_type in [
                'underlying_reported_first_5',
                'underlying_reported_not_stop_5', 'underlying_tied_first_5'
        ]:
            for i in range(5):
                assert data_canonical['data_check'][state][sim_type][i][
                    'n'] == data_test['data_check'][state][sim_type][i]['n']
                assert data_canonical['data_check'][state][sim_type][i][
                    'k'] == data_test['data_check'][state][sim_type][i]['k']
                assert data_canonical['data_check'][state][sim_type][i][
                    'kmin'] == data_test['data_check'][state][sim_type][i][
                        'kmin']
                assert data_canonical['data_check'][state][sim_type][i][
                    'stop'] == data_test['data_check'][state][sim_type][i][
                        'stop']
                assert data_canonical['data_check'][state][sim_type][i]['next_round_size'] == \
                    data_test['data_check'][state][sim_type][i]['next_round_size']
                assert abs(data_canonical['data_check'][state][sim_type][i]
                           ['p_value'] - data_test['data_check'][state]
                           [sim_type][i]['p_value']) < .000001
                assert abs(data_canonical['data_check'][state][sim_type][i]
                           ['next_round_sprob'] - data_test['data_check']
                           [state][sim_type][i]['next_round_sprob']) < .000001