def read_audit_spec_contest(e, args): """ Read 3-audit/31-audit-spec/audit-spec-contest.csv """ election_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT, e.election_dirname) audit_spec_pathname = os.path.join(election_pathname, "3-audit", "31-audit-spec") filename = utils.greatest_name(audit_spec_pathname, "audit-spec-contest", ".csv") file_pathname = os.path.join(audit_spec_pathname, filename) fieldnames = [ "Measurement id", "Contest", "Risk Measurement Method", "Risk Limit", "Risk Upset Threshold", "Sampling Mode", "Initial Status", "Param 1", "Param 2" ] rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False) logger.info("read_audit_spec_contest: e.mid: %s", e.mids) for row in rows: mid = row["Measurement id"] e.mids.append(mid) e.cid_m[mid] = row["Contest"] e.risk_method_m[mid] = row["Risk Measurement Method"] e.risk_limit_m[mid] = float(row["Risk Limit"]) e.risk_upset_m[mid] = float(row["Risk Upset Threshold"]) e.sampling_mode_m[mid] = row["Sampling Mode"] e.initial_status_m[mid] = row["Initial Status"] e.risk_measurement_parameters_m[mid] = (row["Param 1"], row["Param 2"])
def read_election_spec_contests(e): """ Read file election-spec-contests.csv, put results into Election e. """ election_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT, e.election_dirname) spec_pathname = os.path.join(election_pathname, "1-election-spec") filename = utils.greatest_name(spec_pathname, "election-spec-contests", ".csv") file_pathname = os.path.join(spec_pathname, filename) fieldnames = [ "Contest", "Contest type", "Params", "Write-ins", "Selections" ] rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True) for row in rows: cid = row["Contest"] e.cids.append(cid) e.contest_type_c[cid] = row["Contest type"].lower() e.params_c[cid] = row["Params"] e.write_ins_c[cid] = row["Write-ins"].lower() e.selids_c[cid] = {} for selid in row["Selections"]: e.selids_c[cid][selid] = True
def read_reported_cvrs(e): """ Read reported votes 22-reported-cvrs/reported-cvrs-PBCID.csv. """ election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname) specification_pathname = os.path.join(election_pathname, "2-reported", "22-reported-cvrs") fieldnames = [ "Collection", "Scanner", "Ballot id", "Contest", "Selections" ] for pbcid in e.pbcids: safe_pbcid = ids.filename_safe(pbcid) filename = utils.greatest_name(specification_pathname, "reported-cvrs-" + safe_pbcid, ".csv") file_pathname = os.path.join(specification_pathname, filename) rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True) for row in rows: pbcid = row["Collection"] scanner = row["Scanner"] bid = row["Ballot id"] cid = row["Contest"] vote = row["Selections"] vote = tuple(sorted(vote)) # put vote selids into canonical order utils.nested_set(e.rv_cpb, [cid, pbcid, bid], vote) utils.nested_set(e.votes_c, [cid, vote], True)
def read_reported_ballot_manifests(e): """ Read ballot manifest file 21-reported-ballot-manifests and expand rows if needed. """ election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname) specification_pathname = os.path.join(election_pathname, "2-reported", "21-reported-ballot-manifests") fieldnames = [ "Collection", "Box", "Position", "Stamp", "Ballot id", "Number of ballots", "Required Contests", "Possible Contests", "Comments" ] for pbcid in e.pbcids: safe_pbcid = ids.filename_safe(pbcid) filename = utils.greatest_name(specification_pathname, "manifest-" + safe_pbcid, ".csv") file_pathname = os.path.join(specification_pathname, filename) rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False) for row in rows: pbcid = row["Collection"] boxid = row["Box"] position = row["Position"] stamp = row["Stamp"] bid = row["Ballot id"] try: num = int(row["Number of ballots"]) except ValueError: utils.myerror( "Number {} of ballots not an integer.".format(num)) if num <= 0: utils.mywarning( "Number {} of ballots not positive.".format(num)) req = row["Required Contests"] poss = row["Possible Contests"] comments = row["Comments"] bids = utils.count_on(bid, num) stamps = utils.count_on(stamp, num) positions = utils.count_on(position, num) for i in range(num): # utils.nested_set(e.bids_p, [pbcid, bids[i]], True) if pbcid not in e.bids_p: e.bids_p[pbcid] = [] e.bids_p[pbcid].append(bids[i]) utils.nested_set(e.boxid_pb, [pbcid, bids[i]], boxid) utils.nested_set(e.position_pb, [pbcid, bids[i]], position[i]) utils.nested_set(e.stamp_pb, [pbcid, bids[i]], stamps[i]) utils.nested_set(e.required_gid_pb, [pbcid, bids[i]], req) utils.nested_set(e.possible_gid_pb, [pbcid, bids[i]], poss) utils.nested_set(e.comments_pb, [pbcid, bids[i]], comments)
def read_saved_state(e): """ Read state from latest 3-audit/34-audit-output/audit-output-saved-state.json """ dirpath = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname, "3-audit", "34-audit-output") filename = utils.greatest_name(dirpath, "audit-output-saved-state", ".json") file_pathname = os.path.join(dirpath, filename) file = open(file_pathname, "r") e.saved_state = json.load(file)
def read_reported_outcomes(e): election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname) specification_pathname = os.path.join(election_pathname, "2-reported") fieldnames = ["Contest", "Winner(s)"] filename = utils.greatest_name(specification_pathname, "23-reported-outcomes", ".csv") file_pathname = os.path.join(specification_pathname, filename) rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True) for row in rows: cid = row["Contest"] winners = row["Winner(s)"] utils.nested_set(e.ro_c, [cid], winners)
def read_audit_spec_collection(e, args): """ Read 3-audit/31-audit-spec/audit-spec-collection.csv """ election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname) audit_spec_pathname = os.path.join(election_pathname, "3-audit", "31-audit-spec") filename = utils.greatest_name(audit_spec_pathname, "audit-spec-collection", ".csv") file_pathname = os.path.join(audit_spec_pathname, filename) fieldnames = ["Collection", "Max audit rate"] rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False) for row in rows: pbcid = row["Collection"] e.max_audit_rate_p[pbcid] = int(row["Max audit rate"])
def read_syn2_csv(e, synpar): """ Read file defining syn2 synthetic election spec. """ syn2_pathname = os.path.join(multi.ELECTIONS_ROOT, "syn2_specs") filename = utils.greatest_name(syn2_pathname, synpar.election_dirname, ".csv") file_pathname = os.path.join(syn2_pathname, filename) fieldnames = [ "Contest", "Collection", "Reported Vote", "Actual Vote", "Number" ] rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False) return [(row["Contest"], row["Collection"], (row["Reported Vote"], ), (row["Actual Vote"], ), row["Number"]) for row in rows]
def read_audit_spec_global(e, args): """ Read 3-audit/31-audit-spec/audit-spec-global.csv """ election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname) audit_spec_pathname = os.path.join(election_pathname, "3-audit", "31-audit-spec") filename = utils.greatest_name(audit_spec_pathname, "audit-spec-global", ".csv") file_pathname = os.path.join(audit_spec_pathname, filename) fieldnames = ["Global Audit Parameter", "Value"] rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False) for row in rows: parameter = row["Global Audit Parameter"] value = row["Value"] if parameter == "Max audit stage time": e.max_stage_time = value
def counting(prefix, dirpath="./elections"): results = dd(int) output_prefix = "audit-output-contest-status" output_postfix = ".csv" output_inner_dir = path.join("3-audit", "34-audit-output") output_fields = [ "Measurement id", "Contest", "Risk Measurement Method", "Risk Limit", "Risk Upset Threshold", "Sampling Mode", "Status", "Param 1", "Param 2" ] for election_dir in os.listdir(dirpath): # Find the correct directory if election_dir.startswith(prefix): election_dir = path.join(dirpath, election_dir) output_dir = path.join(election_dir, output_inner_dir) if path.exists(output_dir): file = greatest_name(output_dir, output_prefix, output_postfix) file = path.join(output_dir, file) field_dict = read_csv_file(file, required_fieldnames=output_fields) print(file) print(field_dict) for row in field_dict: results[row["Status"]] += 1 result_path = path.join(".", "results") if not path.exists(result_path): os.makedirs(result_path) all_status = ["Upset", "Passed", "Open"] result_csv = path.join(result_path, prefix + ".csv") try: with open(result_csv, 'w') as csv_file: writer = csv.DictWriter(csv_file, fieldnames=all_status) writer.writeheader() for data in [ results, ]: writer.writerow(data) except IOError: print("I/O error") return results
def read_audit_spec_seed(e, args): """ Read audit seed from 3-audit/31-audit-spec/audit-spec-seed.csv Do not overwrite e.audit_seed if it was non-None because this means it was already set from the command line. """ election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname) audit_spec_pathname = os.path.join(election_pathname, "3-audit", "31-audit-spec") filename = utils.greatest_name(audit_spec_pathname, "audit-spec-seed", ".csv") file_pathname = os.path.join(audit_spec_pathname, filename) fieldnames = ["Audit seed"] rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False) for row in rows: new_audit_seed = row["Audit seed"] if e.audit_seed == None: set_audit_seed(e, new_audit_seed)
def read_election_spec_contest_groups(e): """ Read file election-spec-contest-groups.csv, put results into Election e. """ election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname) spec_pathname = os.path.join(election_pathname, "1-election-spec") filename = utils.greatest_name(spec_pathname, "election-spec-contest-groups", ".csv") file_pathname = os.path.join(spec_pathname, filename) fieldnames = ["Contest group", "Contest(s) or group(s)"] rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True) for row in rows: gid = row["Contest group"] if gid in e.cids: raise ValueError( "Contest group id {} must not also be a contest id.".format( gid)) e.gids.append(gid) e.cgids_g[gid] = row["Contest(s) or group(s)"]
def read_meta_csv(e, synpar): """ Read file defining syn3 synthetic election spec. """ syn3_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT, "syn3_specs", synpar.election_dirname) try: # Use config dir if properly defined if synpar.config_dirname is not None: syn3_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT, "syn3_specs", synpar.config_dirname) except NameError: pass filename = utils.greatest_name(syn3_pathname, "meta", ".csv") file_pathname = os.path.join(syn3_pathname, filename) fieldnames = [ "Contest", "Collection", "Winner", "Audit Rate", "Contestants" ] rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True) return [(row["Contest"], row["Collection"], (row["Winner"], ), row["Audit Rate"], row["Contestants"]) for row in rows]
def read_election_spec_general(e, election_dirname): """ Read file 1-election-spec/election-spec-general.csv, put results into Election e. election_dirname is the name of the directory for the election (e.g. "CO-2017-11") with ELECTIONS_ROOT """ election_pathname = os.path.join(multi.ELECTIONS_ROOT, election_dirname) spec_pathname = os.path.join(election_pathname, "1-election-spec") filename = utils.greatest_name(spec_pathname, "election-spec-general", ".csv") file_pathname = os.path.join(spec_pathname, filename) fieldnames = ["Attribute", "Value"] rows = csv_readers.read_csv_file(file_pathname, fieldnames) for row in rows: if "Election name" == row["Attribute"]: e.election_name = row["Value"] elif "Election dirname" == row["Attribute"]: new_election_dirname = row["Value"] if new_election_dirname != election_dirname: utils.mywarning( ("Inconsistent election_dirname {}" "ignored in file {}.").format(new_election_dirname, file_pathname)) else: pass # everything OK elif "Election date" == row["Attribute"]: e.election_date = row["Value"] elif "Election URL" == row["Attribute"]: e.election_url = row["Value"] for attribute in [ "election_name", "election_dirname", "election_date", "election_url" ]: if attribute not in vars(e): utils.mywarning( "Attribute {} not present in 11-general.csv.".format( attribute)) if utils.warnings_given > 0: utils.myerror("Too many errors; terminating.")
def read_audited_votes(e): """ Read audited votes from 3-audit/33-audited-votes/audited-votes-PBCID.csv """ election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname) audited_votes_pathname = os.path.join(election_pathname, "3-audit", "33-audited-votes") for pbcid in e.pbcids: safe_pbcid = ids.filename_safe(pbcid) filename = utils.greatest_name(audited_votes_pathname, "audited-votes-" + safe_pbcid, ".csv") file_pathname = os.path.join(audited_votes_pathname, filename) fieldnames = ["Collection", "Ballot id", "Contest", "Selections"] rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True) for row in rows: pbcid = row["Collection"] bid = row["Ballot id"] cid = row["Contest"] vote = row["Selections"] utils.nested_set(e.av_cpb, [cid, pbcid, bid], vote)
def read_vote_csv(e, synpar, reported=False): """ Read file defining syn3 synthetic election spec. """ syn3_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT, "syn3_specs", synpar.election_dirname) try: # Use config dir if properly defined if synpar.config_dirname is not None: syn3_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT, "syn3_specs", synpar.config_dirname) except NameError: pass filename = utils.greatest_name(syn3_pathname, "reported" if reported else "actual", ".csv") file_pathname = os.path.join(syn3_pathname, filename) fieldnames = ["Contest", "Collection", "Number", "Votes"] rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True) return [(row["Contest"], row["Collection"], row["Number"], row["Votes"]) for row in rows]
def read_election_spec_collections(e): """ Read file 14-collections.csv, put results into Election e. """ election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname) spec_pathname = os.path.join(election_pathname, "1-election-spec") filename = utils.greatest_name(spec_pathname, "election-spec-collections", ".csv") file_pathname = os.path.join(spec_pathname, filename) fieldnames = [ "Collection", "Manager", "CVR type", "Required Contests", "Possible Contests" ] rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False) for row in rows: pbcid = row["Collection"] e.pbcids.append(pbcid) e.manager_p[pbcid] = row["Manager"] e.cvr_type_p[pbcid] = row["CVR type"] e.required_gid_p[pbcid] = row["Required Contests"] e.possible_gid_p[pbcid] = row["Possible Contests"]