def experiment_11(printing_wanted=True):
print "Experiment 11. Number of ballots audited for no-error example from Checkoway et al., Section 4.3 (49)"
print "n=100,000 ballots, m ranges from 0.5% to 5%, 100 simulated audits for each m"
n=100000
#L = [ (1, 1, 0), (1, 2, 0), (1, 3, 200),
# (2, 1, 200), (2, 2, 4800), (2, 3, 100),
# (3, 1, 0), (3, 2, 0), (3, 3, 4700) ]
m_list = tuple(0.01*m for m in [0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5])
epsilon = 0.01
print "epsilon = ",epsilon
num_trials = 100
for m in m_list:
L = [ (1, 1, int(0.1*n)), (1, 2, 0), (1, 3, 0),
(2, 1, 0), (2, 2, int(0.45*n-0.5*m*n)), (2, 3, 0),
(3, 1, 0), (3, 2, 0), (3, 3, int(0.45*n+0.5*m*n))]
for audit_type in ["N","P","NP"]:
num_audited=0
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10])
#schedule=bayes.make_schedule(n,[1,2])
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type=audit_type);
t2=time.time()
num_audited = num_audited+s
if printing_wanted:
print "Reported outcome is "+result+" after examining %d ballots"%s
print "Done in %g seconds."%(t2-t1)
avg_num_audited = num_audited / float(num_trials)
print "m=%7.4f audit_type=%2s avg_num_audited=%5d"%(m,audit_type,avg_num_audited)
def experiment_14(printing_wanted=True):
print "Experiment 14. Number of ballots audited for bidirectional errors with 2-errors example from Checkoway et al., Section 4.3 (52)"
print "n=100,000 ballots, m ranges from 0.5% to 5%, 100 simulated audits for each m"
n=100000
#L = [ (1, 1, 0), (1, 2, 0), (1, 3, 200),
# (2, 1, 200), (2, 2, 4800), (2, 3, 100),
# (3, 1, 0), (3, 2, 0), (3, 3, 4700) ]
m_list = tuple(0.01*m for m in [0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5])
epsilon = 0.01
error = 16/float(100000)
num_trials = 100
print "margin m avg num ballots audited"
print "-------- -----------------------"
for m in m_list:
L = [ (1, 1, int(0.1*n-4*error*n)), (1, 2, int(3*error*n)), (1, 3, int(3*error*n)),
(2, 1, int(2*error*n)), (2, 2, int(0.45*n-0.5*(m+7*error)*n)), (2, 3, 0),
(3, 1, int(2*error*n)), (3, 2, 0), (3, 3, int(0.45*n+0.5*(m-7*error)*n))]
num_audited=0
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10])
#schedule = bayes.make_schedule(n,[1,2])
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P");
t2=time.time()
num_audited = num_audited+s
if printing_wanted:
print "Reported outcome is "+result+" after examining %d ballots"%s
print "Done in %g seconds."%(t2-t1)
avg_num_audited = num_audited / float(num_trials)
print "%7.4f %d"%(m,avg_num_audited)
def experiment_9(printing_wanted=True):
print "Experiment 9. Miscertification rates on example from Checkoway et al., Section 4.1 (48)"
print "Scaled down to n=1000 ballots, m ranges from 0.5% to 5%, 1000 simulations for each m"
n=1000
#L = [ (1, 1, 0), (1, 2, 0), (1, 3, 200),
# (2, 1, 200), (2, 2, 4800), (2, 3, 100),
# (3, 1, 0), (3, 2, 0), (3, 3, 4700) ]
m_list = tuple(0.01*m for m in [1, 2, 3, 4, 5])
epsilon_list = [ 0.00, 0.01, 0.02, 0.05, 0.07, 0.10 ] # added 0.00 rlr 5/14
epsilon_list = [ 0.10, 0.07, 0.05, 0.02, 0.01, 0.00 ] # reverse so we see interesting results first
num_trials=1000
for epsilon in epsilon_list:
print "epsilon=%7.4f"%epsilon
for m in m_list:
print "m=%7.4f"%m
L = [ (1, 1, 0), (1, 2, 0), (1, 3, int(0.4*m*n)),
(2, 1, int(0.4*m*n)), (2, 2, int(0.5*n-0.4*m*n)), (2, 3, int(0.2*m*n)),
(3, 1, 0), (3, 2, 0), (3, 3, int(0.5*n-0.6*m*n))]
count_ok=0
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
#schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10])
schedule=bayes.make_schedule(n,[1,2])
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P");
t2=time.time()
if printing_wanted:
print "Reported outcome is",result,"with sample of size",s
print "Done in %g seconds."%(t2-t1)
if result=="OK":
count_ok = count_ok+1
print "epsilon = %7.4f, m = %7.4f, number of miscertifications = %d (out of %d trials)"%(epsilon,m,count_ok,num_trials)
def experiment_10(printing_wanted=True):
if printing_wanted:
print "Experiment 10. Miscertification rates on a near tie."
L = [(1,1,4999), (1,2, 2), # increased 10x rlr 5/18
(2,1, 0), (2,2,4999)]
print "L = ",L
epsilon_list = [ 0.00, 0.01, 0.02, 0.05, 0.07, 0.1 ] # added 0.00 rlr 5/14
num_trials=1000
seed = 11
print "random number seed = ",seed
for epsilon in epsilon_list:
for audit_type in ["N","P","NP"]:
random.seed(seed) # fix seed, for reproducible results
count_ok = 0
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10])
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type=audit_type);
t2=time.time()
if printing_wanted:
print "trial:",i
print "Reported outcome is",result
print "Done in %g seconds."%(t2-t1)
if result=="OK":
count_ok = count_ok+1
print "For audit type %2s, epsilon = %2.2f, number of miscertifications = %d (out of %d trials)"%(audit_type,epsilon,count_ok,num_trials)
def experiment_19(printing_wanted=True):
print "Experiment 19. Comparison with Stark's comparison audit in Stanislaus Oakdale Measure O, risk limit alpha = 10%"
print "Assumes no errors in reported results, ignores undervotes."
print "100 simulated audits"
epsilon = 0.1
num_trials = 100
L = [ (1, 1, 1728), (1, 2, 0), (2, 1, 0), (2, 2, 1392) ]
num_audited=0
s_list = [ ]
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,range(1,n+1))
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,True,audit_type="N");
t2=time.time()
num_audited = num_audited+s
s_list.append(s)
if printing_wanted:
print "Reported outcome is "+result+" after examining %d ballots"%s
print "Done in %g seconds."%(t2-t1)
print "%d"%s
avg_num_audited = num_audited / float(num_trials)
print "Average number of ballots audited = %7.2f"%avg_num_audited
s_list = sorted(s_list)
print "Median number of ballots audited = %7.2f"%s_list[len(s_list)/2]
n49 = len([x for x in s_list if x<49])
print "Fraction of time bayes audit examines less than 49 ballots:", float(n49)/float(num_trials)
def experiment_21(printing_wanted=True):
print "Experiment 21: Miscertification on IRV example"
import vs # voting system code
A = [ 1, 2, 3 ] # candidates (alternatives)
params = vs.default_params() # parameters (nothing special needed)
vs.setup_TB(A,params) # establish tie-breaker values
B123 = 1 # ballot types in order as generated by vs.perms
B132 = 2
B213 = 3
B231 = 4
B312 = 5
B321 = 6
err = 1 # with err=1 we have an incorrect election
#L = [ (B123,B123,100), (B123,B132,5*err),
# (B132,B132,100), (B132,B123,6*err),
# (B213,B213,120), (B213,B123,4*err),
# (B231,B231, 90), (B231,B213,5*err),
# (B312,B312,100), (B312,B132,5*err),
# (B321,B321,100), (B321,B312,10*err) ]
L = [ (B123, B123, 100), (B123, B132, 20),
(B132, B132, 100), (B132, B123, 24),
(B213, B213, 120), (B213, B123, 16),
(B231, B231, 90), (B231, B213, 20),
(B312, B312, 100), (B312, B132, 20),
(B321, B321, 100), (B321, B312, 40) ]
# schedule = bayes.make_schedule(n,range(1,10000))
#epsilon_list = [0.01, 0.02, 0.05, 0.07, 0.10]
epsilon_list = [0.01]
# list of voting system methods implemented in vs.py, just for reference:
# unanimous_winner
# majority_winner
# plurality_winners
# Condorcet_winner
# Borda_winner
# minimax_winner
# Smith_set
# IRV_winner
# Schulze_winner
# following may not work if interface to cvxopt not available
# gt_winner
# set global variable to select which voting system winner computation to use in vs.f_vs
# print "minimax"
# vs.vs_winner = vs.minimax_winner
print "IRV"
vs.vs_winner = vs.IRV_winner
num_trials = 1000
for epsilon in epsilon_list:
count_ok = 0
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,[10,11])
# now do audit
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,ballot_polling=False,f=vs.f_vs,audit_type="P");
t2=time.time()
print "Reported outcome is ",result,"after examining %d ballots"%s
print "Done in %g seconds."%(t2-t1)
if result=="OK":
count_ok = count_ok+1
print "For epsilon = %2.2f, there were %d miscertifications (out of %d trials)"%(epsilon,count_ok,num_trials)
def experiment_17(printing_wanted=True):
print "Experiment 17. Comparison with Stark's ballot-polling audit in Monterey 2011, risk limit alpha = 10%"
print "Assumes no errors in reported results, and combines write-ins and Mancini into one candidate as Stark did."
print "100 simulated audits"
epsilon = 0.1
num_trials = 100
L = [ (1, 1, 1353), (1, 2, 0), (2, 1, 0), (2, 2, 755) ]
num_audited=0
s_list = [ ]
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,range(1,n+1))
t1 = time.time();
audit_type = "N"
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,ballot_polling=True,audit_type=audit_type);
t2=time.time()
num_audited = num_audited+s
s_list.append(s)
if printing_wanted:
print "Reported outcome is "+result+" after examining %d ballots"%s
print "Done in %g seconds."%(t2-t1)
print "%d"%s
avg_num_audited = num_audited / float(num_trials)
print "Average number of ballots audited = %7.2f"%avg_num_audited
s_list = sorted(s_list)
print "Median number of ballots audited = %7.2f"%s_list[len(s_list)/2]
def experiment_1(printing_wanted=True):
if printing_wanted:
print "Experiment 1. Small error data set."
L = [ (1,1,600), (1,2,100), (2,2,200) ]
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10])
epsilon = 0.05
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P");
t2 = time.time()
if printing_wanted:
print "Reported outcome is",result
print "Done in %g seconds."%(t2-t1)
def experiment_6(printing_wanted=True):
if printing_wanted:
print "Experiment 6. Small debugging data set."
L = [ (1,1,4), (1,2,1),
(2,1,2), (2,2,3) ]
epsilon = 0.05
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10])
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P");
t2 = time.time()
if printing_wanted:
print "Reported outcome is",result
print "Done in %g seconds."%(t2-t1)
return result
def experiment_3(printing_wanted=True):
if printing_wanted:
print "Experiment 3. No error data set from Checkoway/Sarwate/Shacham (eqn 49, page 11 m=1000)"
L = [ (1,1,10000), (2,2,40000), (3,3,50000) ]
r,a,t,n = make_profiles(L,printing_wanted)
# schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10])
# epsilon = 0.01
schedule = bayes.make_schedule(n,[10,11])
epsilon = 0.01
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P");
t2 = time.time()
if printing_wanted:
print "Reported outcome is",result
print "Done in %g seconds."%(t2-t1)
def experiment_5(printing_wanted=True):
if printing_wanted:
print "Experiment 5. Data set from Checkoway/Sarwate/Shacham (section 3.4, page 9), divided by 10"
L = [ (1,1,150), (1,2, 40), (1,3, 10),
(2,1, 30), (2,2,4630), (2,3, 20),
(3,1, 60), (3,2, 60), (3,3,5000) ]
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10])
epsilon = 0.01
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P");
t2 = time.time()
if printing_wanted:
print "Reported outcome is",result
print "Done in %g seconds."%(t2-t1)
return result
def experiment_8(printing_wanted=True):
if printing_wanted:
print "Experiment 8. Same as 7, but just one run."
L = [ (1,1,5000), (1,2, 80), (1,3, 5),
(2,1, 10), (2,2,4500), (2,3, 5),
(3,1, 0), (3,2, 0), (3,3,400) ]
epsilon = 0.05
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,range(1,1000,10))
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P");
t2 = time.time()
if printing_wanted:
print "Reported outcome is",result
print "Done in %g seconds."%(t2-t1)
return result
def experiment_15(printing_wanted=True):
print "Experiment 15: IRV test."
import vs # voting system code
A = [ 1, 2, 3 ] # candidates (alternatives)
params = vs.default_params() # parameters (nothing special needed)
vs.setup_TB(A,params) # establish tie-breaker values
B123 = 1 # ballot types in order as generated by vs.perms
B132 = 2
B213 = 3
B231 = 4
B312 = 5
B321 = 6
err = 1 # with err=1 we have an incorrect election
L = [ (B123,B123,1000), (B123,B132,50*err),
(B132,B132,1000), (B132,B123,60*err),
(B213,B213,1200), (B213,B123,40*err),
(B231,B231, 900), (B231,B213,50*err),
(B312,B312,1000), (B312,B132,50*err),
(B321,B321,1000), (B321,B312,100*err) ]
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,[10,11])
# schedule = bayes.make_schedule(n,range(1,10000))
epsilon = 0.001
# list of voting system methods implemented in vs.py, just for reference:
# unanimous_winner
# majority_winner
# plurality_winners
# Condorcet_winner
# Borda_winner
# minimax_winner
# Smith_set
# IRV_winner
# Schulze_winner
# following may not work if interface to cvxopt not available
# gt_winner
# set global variable to select which voting system winner computation to use in vs.f_vs
# print "minimax"
# vs.vs_winner = vs.minimax_winner
print "IRV"
vs.vs_winner = vs.IRV_winner
# now do audit
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,ballot_polling=False,f=vs.f_vs,audit_type="P");
t2=time.time()
if printing_wanted:
print "Reported outcome is ",result,"after examining %d ballots"%s
print "Done in %g seconds."%(t2-t1)
def experiment_16(printing_wanted=True):
print "Experiment 16. Comparison with Stark's single-ballot audit in San Luis Obispo, Measure A, risk limit alpha = 10%"
print "100 simulated audits"
epsilon = 0.1
num_trials = 100
L = [ (1, 1, 7848), (1, 2, 0), (2, 1, 0), (2, 2, 2764) ]
num_audited=0
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,range(1,n+1))
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P");
t2=time.time()
num_audited = num_audited+s
if printing_wanted:
print "Reported outcome is "+result+" after examining %d ballots"%s
print "Done in %g seconds."%(t2-t1)
avg_num_audited = num_audited / float(num_trials)
print "Average number of ballots audited = %7.2f"%avg_num_audited
def experiment_20(printing_wanted=True):
if printing_wanted:
print "Experiment 20. Miscertification rates on a near tie."
L = [(1,1,499), (1,2,2),
(2,1,0), (2,2,499)]
epsilon_list = [ 0.07 ]
num_trials=1000
for epsilon in epsilon_list:
count_ok = 0
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10])
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P");
t2=time.time()
if printing_wanted:
print "Reported outcome is",result
print "Done in %g seconds."%(t2-t1)
if result=="OK":
count_ok = count_ok+1
print "For epsilon = %2.2f, there were %d miscertifications (out of %d trials)"%(epsilon,count_ok,num_trials)
def experiment_18(printing_wanted=True):
print "Experiment 18. Single ballot comparison audit on Monterey 2011, Water Management District Director, risk limit alpha = 10%"
print "Assumes no errors in reported results, and combines write-ins and Mancini into one candidate as Stark did in his ballot-polling audit."
print "100 simulated audits"
epsilon = 0.1
num_trials = 100
L = [ (1, 1, 1353), (1, 2, 0), (2, 1, 0), (2, 2, 755) ]
num_audited=0
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,range(1,n+1))
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P");
t2=time.time()
num_audited = num_audited+s
if printing_wanted:
print "Reported outcome is "+result+" after examining %d ballots"%s
print "Done in %g seconds."%(t2-t1)
print "%d"%s
avg_num_audited = num_audited / float(num_trials)
print "Average number of ballots audited = %7.2f"%avg_num_audited
def experiment_23(printing_wanted=True):
if printing_wanted:
print "Experiment 23. To compare with go implementation for ballot polling."
L = [ (1,1,223250), (1,2,225750), (1,3,225200), (1,4,225800) ]
epsilon_list = [ 0.01, 0.02, 0.05, 0.07, 0.10 ]
epsilon_list = [ 0.02 ]
# num_trials=1000
num_trials=1
for epsilon in epsilon_list:
count_ok = 0
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,[4,5])
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="N");
t2=time.time()
if printing_wanted:
print "Reported outcome is",result
print "Done in %g seconds."%(t2-t1)
if result=="OK":
count_ok = count_ok+1
print "miscertification trial",i,"s=",s
print "For epsilon = %2.2f, there were %d miscertifications (out of %d trials)"%(epsilon,count_ok,num_trials)
def experiment_4(printing_wanted=True):
if printing_wanted:
print "Experiment 4. Data set from Checkoway/Sarwate/Shacham (section 3.4, page 9)"
# ---> Note that their matrix is transpose of ours!!
# ---> Ballot type 1 corresponds to "None", whereas 2 and 3 are candidates,
# ---> but this doesn't affect anything here since 1 has no chance of winning.
L = [ (1,1,1500), (1,2, 400), (1,3, 100),
(2,1, 300), (2,2,46300), (2,3, 200),
(3,1, 600), (3,2, 600), (3,3,50000) ]
# with our procedure, audited only 1420 ballots
# (with epsilon = 0.01) rather than 2496 ballots for CSS.
r,a,t,n = make_profiles(L,printing_wanted)
# schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10])
schedule = bayes.make_schedule(n,[10,11])
epsilon = 0.01
ballot_polling = True
t1 = time.time();
(result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,ballot_polling,audit_type="NP20");
t2 = time.time()
if printing_wanted:
print "Reported outcome is",result
print "Done in %g seconds."%(t2-t1)
return result
def experiment_22(printing_wanted=True):
if printing_wanted:
print "Experiment 22. Miscertification rates on another toy example."
L = [ (1,1,401), (1,2, 50), (1,3, 50),
(2,1, 50), (2,2,350), (2,3, 0),
(3,1, 0), (3,2,100), (3,3, 0) ]
epsilon_list = [ 0.01, 0.02, 0.05, 0.07, 0.10 ]
# num_trials=1000
num_trials=100
for epsilon in epsilon_list:
count_ok = 0
for i in range(num_trials):
r,a,t,n = make_profiles(L,printing_wanted)
schedule = bayes.make_schedule(n,[10,11])
t1 = time.time();
(result,s)=bayes.audit_dirichlet(r,a,t,epsilon,schedule,printing_wanted,audit_type="P");
t2=time.time()
if printing_wanted:
print "Reported outcome is",result
print "Done in %g seconds."%(t2-t1)
if result=="OK":
count_ok = count_ok+1
print "miscertification trial",i,"s=",s
print "For epsilon = %2.2f, there were %d miscertifications (out of %d trials)"%(epsilon,count_ok,num_trials)