def experiment_11(printing_wanted=True): print "Experiment 11. Number of ballots audited for no-error example from Checkoway et al., Section 4.3 (49)" print "n=100,000 ballots, m ranges from 0.5% to 5%, 100 simulated audits for each m" n=100000 #L = [ (1, 1, 0), (1, 2, 0), (1, 3, 200), # (2, 1, 200), (2, 2, 4800), (2, 3, 100), # (3, 1, 0), (3, 2, 0), (3, 3, 4700) ] m_list = tuple(0.01*m for m in [0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5]) epsilon = 0.01 print "epsilon = ",epsilon num_trials = 100 for m in m_list: L = [ (1, 1, int(0.1*n)), (1, 2, 0), (1, 3, 0), (2, 1, 0), (2, 2, int(0.45*n-0.5*m*n)), (2, 3, 0), (3, 1, 0), (3, 2, 0), (3, 3, int(0.45*n+0.5*m*n))] for audit_type in ["N","P","NP"]: num_audited=0 for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10]) #schedule=bayes.make_schedule(n,[1,2]) t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type=audit_type); t2=time.time() num_audited = num_audited+s if printing_wanted: print "Reported outcome is "+result+" after examining %d ballots"%s print "Done in %g seconds."%(t2-t1) avg_num_audited = num_audited / float(num_trials) print "m=%7.4f audit_type=%2s avg_num_audited=%5d"%(m,audit_type,avg_num_audited)
def experiment_14(printing_wanted=True): print "Experiment 14. Number of ballots audited for bidirectional errors with 2-errors example from Checkoway et al., Section 4.3 (52)" print "n=100,000 ballots, m ranges from 0.5% to 5%, 100 simulated audits for each m" n=100000 #L = [ (1, 1, 0), (1, 2, 0), (1, 3, 200), # (2, 1, 200), (2, 2, 4800), (2, 3, 100), # (3, 1, 0), (3, 2, 0), (3, 3, 4700) ] m_list = tuple(0.01*m for m in [0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5]) epsilon = 0.01 error = 16/float(100000) num_trials = 100 print "margin m avg num ballots audited" print "-------- -----------------------" for m in m_list: L = [ (1, 1, int(0.1*n-4*error*n)), (1, 2, int(3*error*n)), (1, 3, int(3*error*n)), (2, 1, int(2*error*n)), (2, 2, int(0.45*n-0.5*(m+7*error)*n)), (2, 3, 0), (3, 1, int(2*error*n)), (3, 2, 0), (3, 3, int(0.45*n+0.5*(m-7*error)*n))] num_audited=0 for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10]) #schedule = bayes.make_schedule(n,[1,2]) t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P"); t2=time.time() num_audited = num_audited+s if printing_wanted: print "Reported outcome is "+result+" after examining %d ballots"%s print "Done in %g seconds."%(t2-t1) avg_num_audited = num_audited / float(num_trials) print "%7.4f %d"%(m,avg_num_audited)
def experiment_9(printing_wanted=True): print "Experiment 9. Miscertification rates on example from Checkoway et al., Section 4.1 (48)" print "Scaled down to n=1000 ballots, m ranges from 0.5% to 5%, 1000 simulations for each m" n=1000 #L = [ (1, 1, 0), (1, 2, 0), (1, 3, 200), # (2, 1, 200), (2, 2, 4800), (2, 3, 100), # (3, 1, 0), (3, 2, 0), (3, 3, 4700) ] m_list = tuple(0.01*m for m in [1, 2, 3, 4, 5]) epsilon_list = [ 0.00, 0.01, 0.02, 0.05, 0.07, 0.10 ] # added 0.00 rlr 5/14 epsilon_list = [ 0.10, 0.07, 0.05, 0.02, 0.01, 0.00 ] # reverse so we see interesting results first num_trials=1000 for epsilon in epsilon_list: print "epsilon=%7.4f"%epsilon for m in m_list: print "m=%7.4f"%m L = [ (1, 1, 0), (1, 2, 0), (1, 3, int(0.4*m*n)), (2, 1, int(0.4*m*n)), (2, 2, int(0.5*n-0.4*m*n)), (2, 3, int(0.2*m*n)), (3, 1, 0), (3, 2, 0), (3, 3, int(0.5*n-0.6*m*n))] count_ok=0 for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) #schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10]) schedule=bayes.make_schedule(n,[1,2]) t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P"); t2=time.time() if printing_wanted: print "Reported outcome is",result,"with sample of size",s print "Done in %g seconds."%(t2-t1) if result=="OK": count_ok = count_ok+1 print "epsilon = %7.4f, m = %7.4f, number of miscertifications = %d (out of %d trials)"%(epsilon,m,count_ok,num_trials)
def experiment_10(printing_wanted=True): if printing_wanted: print "Experiment 10. Miscertification rates on a near tie." L = [(1,1,4999), (1,2, 2), # increased 10x rlr 5/18 (2,1, 0), (2,2,4999)] print "L = ",L epsilon_list = [ 0.00, 0.01, 0.02, 0.05, 0.07, 0.1 ] # added 0.00 rlr 5/14 num_trials=1000 seed = 11 print "random number seed = ",seed for epsilon in epsilon_list: for audit_type in ["N","P","NP"]: random.seed(seed) # fix seed, for reproducible results count_ok = 0 for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10]) t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type=audit_type); t2=time.time() if printing_wanted: print "trial:",i print "Reported outcome is",result print "Done in %g seconds."%(t2-t1) if result=="OK": count_ok = count_ok+1 print "For audit type %2s, epsilon = %2.2f, number of miscertifications = %d (out of %d trials)"%(audit_type,epsilon,count_ok,num_trials)
def experiment_19(printing_wanted=True): print "Experiment 19. Comparison with Stark's comparison audit in Stanislaus Oakdale Measure O, risk limit alpha = 10%" print "Assumes no errors in reported results, ignores undervotes." print "100 simulated audits" epsilon = 0.1 num_trials = 100 L = [ (1, 1, 1728), (1, 2, 0), (2, 1, 0), (2, 2, 1392) ] num_audited=0 s_list = [ ] for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,range(1,n+1)) t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,True,audit_type="N"); t2=time.time() num_audited = num_audited+s s_list.append(s) if printing_wanted: print "Reported outcome is "+result+" after examining %d ballots"%s print "Done in %g seconds."%(t2-t1) print "%d"%s avg_num_audited = num_audited / float(num_trials) print "Average number of ballots audited = %7.2f"%avg_num_audited s_list = sorted(s_list) print "Median number of ballots audited = %7.2f"%s_list[len(s_list)/2] n49 = len([x for x in s_list if x<49]) print "Fraction of time bayes audit examines less than 49 ballots:", float(n49)/float(num_trials)
def experiment_21(printing_wanted=True): print "Experiment 21: Miscertification on IRV example" import vs # voting system code A = [ 1, 2, 3 ] # candidates (alternatives) params = vs.default_params() # parameters (nothing special needed) vs.setup_TB(A,params) # establish tie-breaker values B123 = 1 # ballot types in order as generated by vs.perms B132 = 2 B213 = 3 B231 = 4 B312 = 5 B321 = 6 err = 1 # with err=1 we have an incorrect election #L = [ (B123,B123,100), (B123,B132,5*err), # (B132,B132,100), (B132,B123,6*err), # (B213,B213,120), (B213,B123,4*err), # (B231,B231, 90), (B231,B213,5*err), # (B312,B312,100), (B312,B132,5*err), # (B321,B321,100), (B321,B312,10*err) ] L = [ (B123, B123, 100), (B123, B132, 20), (B132, B132, 100), (B132, B123, 24), (B213, B213, 120), (B213, B123, 16), (B231, B231, 90), (B231, B213, 20), (B312, B312, 100), (B312, B132, 20), (B321, B321, 100), (B321, B312, 40) ] # schedule = bayes.make_schedule(n,range(1,10000)) #epsilon_list = [0.01, 0.02, 0.05, 0.07, 0.10] epsilon_list = [0.01] # list of voting system methods implemented in vs.py, just for reference: # unanimous_winner # majority_winner # plurality_winners # Condorcet_winner # Borda_winner # minimax_winner # Smith_set # IRV_winner # Schulze_winner # following may not work if interface to cvxopt not available # gt_winner # set global variable to select which voting system winner computation to use in vs.f_vs # print "minimax" # vs.vs_winner = vs.minimax_winner print "IRV" vs.vs_winner = vs.IRV_winner num_trials = 1000 for epsilon in epsilon_list: count_ok = 0 for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,[10,11]) # now do audit t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,ballot_polling=False,f=vs.f_vs,audit_type="P"); t2=time.time() print "Reported outcome is ",result,"after examining %d ballots"%s print "Done in %g seconds."%(t2-t1) if result=="OK": count_ok = count_ok+1 print "For epsilon = %2.2f, there were %d miscertifications (out of %d trials)"%(epsilon,count_ok,num_trials)
def experiment_17(printing_wanted=True): print "Experiment 17. Comparison with Stark's ballot-polling audit in Monterey 2011, risk limit alpha = 10%" print "Assumes no errors in reported results, and combines write-ins and Mancini into one candidate as Stark did." print "100 simulated audits" epsilon = 0.1 num_trials = 100 L = [ (1, 1, 1353), (1, 2, 0), (2, 1, 0), (2, 2, 755) ] num_audited=0 s_list = [ ] for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,range(1,n+1)) t1 = time.time(); audit_type = "N" (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,ballot_polling=True,audit_type=audit_type); t2=time.time() num_audited = num_audited+s s_list.append(s) if printing_wanted: print "Reported outcome is "+result+" after examining %d ballots"%s print "Done in %g seconds."%(t2-t1) print "%d"%s avg_num_audited = num_audited / float(num_trials) print "Average number of ballots audited = %7.2f"%avg_num_audited s_list = sorted(s_list) print "Median number of ballots audited = %7.2f"%s_list[len(s_list)/2]
def experiment_1(printing_wanted=True): if printing_wanted: print "Experiment 1. Small error data set." L = [ (1,1,600), (1,2,100), (2,2,200) ] r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10]) epsilon = 0.05 t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P"); t2 = time.time() if printing_wanted: print "Reported outcome is",result print "Done in %g seconds."%(t2-t1)
def experiment_6(printing_wanted=True): if printing_wanted: print "Experiment 6. Small debugging data set." L = [ (1,1,4), (1,2,1), (2,1,2), (2,2,3) ] epsilon = 0.05 r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10]) t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P"); t2 = time.time() if printing_wanted: print "Reported outcome is",result print "Done in %g seconds."%(t2-t1) return result
def experiment_3(printing_wanted=True): if printing_wanted: print "Experiment 3. No error data set from Checkoway/Sarwate/Shacham (eqn 49, page 11 m=1000)" L = [ (1,1,10000), (2,2,40000), (3,3,50000) ] r,a,t,n = make_profiles(L,printing_wanted) # schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10]) # epsilon = 0.01 schedule = bayes.make_schedule(n,[10,11]) epsilon = 0.01 t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P"); t2 = time.time() if printing_wanted: print "Reported outcome is",result print "Done in %g seconds."%(t2-t1)
def experiment_5(printing_wanted=True): if printing_wanted: print "Experiment 5. Data set from Checkoway/Sarwate/Shacham (section 3.4, page 9), divided by 10" L = [ (1,1,150), (1,2, 40), (1,3, 10), (2,1, 30), (2,2,4630), (2,3, 20), (3,1, 60), (3,2, 60), (3,3,5000) ] r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10]) epsilon = 0.01 t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P"); t2 = time.time() if printing_wanted: print "Reported outcome is",result print "Done in %g seconds."%(t2-t1) return result
def experiment_8(printing_wanted=True): if printing_wanted: print "Experiment 8. Same as 7, but just one run." L = [ (1,1,5000), (1,2, 80), (1,3, 5), (2,1, 10), (2,2,4500), (2,3, 5), (3,1, 0), (3,2, 0), (3,3,400) ] epsilon = 0.05 r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,range(1,1000,10)) t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P"); t2 = time.time() if printing_wanted: print "Reported outcome is",result print "Done in %g seconds."%(t2-t1) return result
def experiment_15(printing_wanted=True): print "Experiment 15: IRV test." import vs # voting system code A = [ 1, 2, 3 ] # candidates (alternatives) params = vs.default_params() # parameters (nothing special needed) vs.setup_TB(A,params) # establish tie-breaker values B123 = 1 # ballot types in order as generated by vs.perms B132 = 2 B213 = 3 B231 = 4 B312 = 5 B321 = 6 err = 1 # with err=1 we have an incorrect election L = [ (B123,B123,1000), (B123,B132,50*err), (B132,B132,1000), (B132,B123,60*err), (B213,B213,1200), (B213,B123,40*err), (B231,B231, 900), (B231,B213,50*err), (B312,B312,1000), (B312,B132,50*err), (B321,B321,1000), (B321,B312,100*err) ] r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,[10,11]) # schedule = bayes.make_schedule(n,range(1,10000)) epsilon = 0.001 # list of voting system methods implemented in vs.py, just for reference: # unanimous_winner # majority_winner # plurality_winners # Condorcet_winner # Borda_winner # minimax_winner # Smith_set # IRV_winner # Schulze_winner # following may not work if interface to cvxopt not available # gt_winner # set global variable to select which voting system winner computation to use in vs.f_vs # print "minimax" # vs.vs_winner = vs.minimax_winner print "IRV" vs.vs_winner = vs.IRV_winner # now do audit t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,ballot_polling=False,f=vs.f_vs,audit_type="P"); t2=time.time() if printing_wanted: print "Reported outcome is ",result,"after examining %d ballots"%s print "Done in %g seconds."%(t2-t1)
def experiment_16(printing_wanted=True): print "Experiment 16. Comparison with Stark's single-ballot audit in San Luis Obispo, Measure A, risk limit alpha = 10%" print "100 simulated audits" epsilon = 0.1 num_trials = 100 L = [ (1, 1, 7848), (1, 2, 0), (2, 1, 0), (2, 2, 2764) ] num_audited=0 for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,range(1,n+1)) t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P"); t2=time.time() num_audited = num_audited+s if printing_wanted: print "Reported outcome is "+result+" after examining %d ballots"%s print "Done in %g seconds."%(t2-t1) avg_num_audited = num_audited / float(num_trials) print "Average number of ballots audited = %7.2f"%avg_num_audited
def experiment_20(printing_wanted=True): if printing_wanted: print "Experiment 20. Miscertification rates on a near tie." L = [(1,1,499), (1,2,2), (2,1,0), (2,2,499)] epsilon_list = [ 0.07 ] num_trials=1000 for epsilon in epsilon_list: count_ok = 0 for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10]) t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P"); t2=time.time() if printing_wanted: print "Reported outcome is",result print "Done in %g seconds."%(t2-t1) if result=="OK": count_ok = count_ok+1 print "For epsilon = %2.2f, there were %d miscertifications (out of %d trials)"%(epsilon,count_ok,num_trials)
def experiment_18(printing_wanted=True): print "Experiment 18. Single ballot comparison audit on Monterey 2011, Water Management District Director, risk limit alpha = 10%" print "Assumes no errors in reported results, and combines write-ins and Mancini into one candidate as Stark did in his ballot-polling audit." print "100 simulated audits" epsilon = 0.1 num_trials = 100 L = [ (1, 1, 1353), (1, 2, 0), (2, 1, 0), (2, 2, 755) ] num_audited=0 for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,range(1,n+1)) t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="P"); t2=time.time() num_audited = num_audited+s if printing_wanted: print "Reported outcome is "+result+" after examining %d ballots"%s print "Done in %g seconds."%(t2-t1) print "%d"%s avg_num_audited = num_audited / float(num_trials) print "Average number of ballots audited = %7.2f"%avg_num_audited
def experiment_23(printing_wanted=True): if printing_wanted: print "Experiment 23. To compare with go implementation for ballot polling." L = [ (1,1,223250), (1,2,225750), (1,3,225200), (1,4,225800) ] epsilon_list = [ 0.01, 0.02, 0.05, 0.07, 0.10 ] epsilon_list = [ 0.02 ] # num_trials=1000 num_trials=1 for epsilon in epsilon_list: count_ok = 0 for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,[4,5]) t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,audit_type="N"); t2=time.time() if printing_wanted: print "Reported outcome is",result print "Done in %g seconds."%(t2-t1) if result=="OK": count_ok = count_ok+1 print "miscertification trial",i,"s=",s print "For epsilon = %2.2f, there were %d miscertifications (out of %d trials)"%(epsilon,count_ok,num_trials)
def experiment_4(printing_wanted=True): if printing_wanted: print "Experiment 4. Data set from Checkoway/Sarwate/Shacham (section 3.4, page 9)" # ---> Note that their matrix is transpose of ours!! # ---> Ballot type 1 corresponds to "None", whereas 2 and 3 are candidates, # ---> but this doesn't affect anything here since 1 has no chance of winning. L = [ (1,1,1500), (1,2, 400), (1,3, 100), (2,1, 300), (2,2,46300), (2,3, 200), (3,1, 600), (3,2, 600), (3,3,50000) ] # with our procedure, audited only 1420 ballots # (with epsilon = 0.01) rather than 2496 ballots for CSS. r,a,t,n = make_profiles(L,printing_wanted) # schedule = bayes.make_schedule(n,[1,2,3,4,5,6,7,8,9,10]) schedule = bayes.make_schedule(n,[10,11]) epsilon = 0.01 ballot_polling = True t1 = time.time(); (result,s)=bayes.audit(r,a,t,epsilon,schedule,printing_wanted,ballot_polling,audit_type="NP20"); t2 = time.time() if printing_wanted: print "Reported outcome is",result print "Done in %g seconds."%(t2-t1) return result
def experiment_22(printing_wanted=True): if printing_wanted: print "Experiment 22. Miscertification rates on another toy example." L = [ (1,1,401), (1,2, 50), (1,3, 50), (2,1, 50), (2,2,350), (2,3, 0), (3,1, 0), (3,2,100), (3,3, 0) ] epsilon_list = [ 0.01, 0.02, 0.05, 0.07, 0.10 ] # num_trials=1000 num_trials=100 for epsilon in epsilon_list: count_ok = 0 for i in range(num_trials): r,a,t,n = make_profiles(L,printing_wanted) schedule = bayes.make_schedule(n,[10,11]) t1 = time.time(); (result,s)=bayes.audit_dirichlet(r,a,t,epsilon,schedule,printing_wanted,audit_type="P"); t2=time.time() if printing_wanted: print "Reported outcome is",result print "Done in %g seconds."%(t2-t1) if result=="OK": count_ok = count_ok+1 print "miscertification trial",i,"s=",s print "For epsilon = %2.2f, there were %d miscertifications (out of %d trials)"%(epsilon,count_ok,num_trials)