def read_audit_spec_contest(e, args):
""" Read 3-audit/31-audit-spec/audit-spec-contest.csv """
election_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT,
e.election_dirname)
audit_spec_pathname = os.path.join(election_pathname, "3-audit",
"31-audit-spec")
filename = utils.greatest_name(audit_spec_pathname, "audit-spec-contest",
".csv")
file_pathname = os.path.join(audit_spec_pathname, filename)
fieldnames = [
"Measurement id", "Contest", "Risk Measurement Method", "Risk Limit",
"Risk Upset Threshold", "Sampling Mode", "Initial Status", "Param 1",
"Param 2"
]
rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False)
logger.info("read_audit_spec_contest: e.mid: %s", e.mids)
for row in rows:
mid = row["Measurement id"]
e.mids.append(mid)
e.cid_m[mid] = row["Contest"]
e.risk_method_m[mid] = row["Risk Measurement Method"]
e.risk_limit_m[mid] = float(row["Risk Limit"])
e.risk_upset_m[mid] = float(row["Risk Upset Threshold"])
e.sampling_mode_m[mid] = row["Sampling Mode"]
e.initial_status_m[mid] = row["Initial Status"]
e.risk_measurement_parameters_m[mid] = (row["Param 1"], row["Param 2"])
def read_election_spec_contests(e):
"""
Read file election-spec-contests.csv, put results into Election e.
"""
election_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT,
e.election_dirname)
spec_pathname = os.path.join(election_pathname, "1-election-spec")
filename = utils.greatest_name(spec_pathname, "election-spec-contests",
".csv")
file_pathname = os.path.join(spec_pathname, filename)
fieldnames = [
"Contest", "Contest type", "Params", "Write-ins", "Selections"
]
rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True)
for row in rows:
cid = row["Contest"]
e.cids.append(cid)
e.contest_type_c[cid] = row["Contest type"].lower()
e.params_c[cid] = row["Params"]
e.write_ins_c[cid] = row["Write-ins"].lower()
e.selids_c[cid] = {}
for selid in row["Selections"]:
e.selids_c[cid][selid] = True
def read_reported_cvrs(e):
"""
Read reported votes 22-reported-cvrs/reported-cvrs-PBCID.csv.
"""
election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
specification_pathname = os.path.join(election_pathname, "2-reported",
"22-reported-cvrs")
fieldnames = [
"Collection", "Scanner", "Ballot id", "Contest", "Selections"
]
for pbcid in e.pbcids:
safe_pbcid = ids.filename_safe(pbcid)
filename = utils.greatest_name(specification_pathname,
"reported-cvrs-" + safe_pbcid, ".csv")
file_pathname = os.path.join(specification_pathname, filename)
rows = csv_readers.read_csv_file(file_pathname,
fieldnames,
varlen=True)
for row in rows:
pbcid = row["Collection"]
scanner = row["Scanner"]
bid = row["Ballot id"]
cid = row["Contest"]
vote = row["Selections"]
vote = tuple(sorted(vote)) # put vote selids into canonical order
utils.nested_set(e.rv_cpb, [cid, pbcid, bid], vote)
utils.nested_set(e.votes_c, [cid, vote], True)
def read_reported_ballot_manifests(e):
"""
Read ballot manifest file 21-reported-ballot-manifests and expand rows if needed.
"""
election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
specification_pathname = os.path.join(election_pathname, "2-reported",
"21-reported-ballot-manifests")
fieldnames = [
"Collection", "Box", "Position", "Stamp", "Ballot id",
"Number of ballots", "Required Contests", "Possible Contests",
"Comments"
]
for pbcid in e.pbcids:
safe_pbcid = ids.filename_safe(pbcid)
filename = utils.greatest_name(specification_pathname,
"manifest-" + safe_pbcid, ".csv")
file_pathname = os.path.join(specification_pathname, filename)
rows = csv_readers.read_csv_file(file_pathname,
fieldnames,
varlen=False)
for row in rows:
pbcid = row["Collection"]
boxid = row["Box"]
position = row["Position"]
stamp = row["Stamp"]
bid = row["Ballot id"]
try:
num = int(row["Number of ballots"])
except ValueError:
utils.myerror(
"Number {} of ballots not an integer.".format(num))
if num <= 0:
utils.mywarning(
"Number {} of ballots not positive.".format(num))
req = row["Required Contests"]
poss = row["Possible Contests"]
comments = row["Comments"]
bids = utils.count_on(bid, num)
stamps = utils.count_on(stamp, num)
positions = utils.count_on(position, num)
for i in range(num):
# utils.nested_set(e.bids_p, [pbcid, bids[i]], True)
if pbcid not in e.bids_p:
e.bids_p[pbcid] = []
e.bids_p[pbcid].append(bids[i])
utils.nested_set(e.boxid_pb, [pbcid, bids[i]], boxid)
utils.nested_set(e.position_pb, [pbcid, bids[i]], position[i])
utils.nested_set(e.stamp_pb, [pbcid, bids[i]], stamps[i])
utils.nested_set(e.required_gid_pb, [pbcid, bids[i]], req)
utils.nested_set(e.possible_gid_pb, [pbcid, bids[i]], poss)
utils.nested_set(e.comments_pb, [pbcid, bids[i]], comments)
def read_saved_state(e):
"""
Read state from latest 3-audit/34-audit-output/audit-output-saved-state.json
"""
dirpath = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname, "3-audit",
"34-audit-output")
filename = utils.greatest_name(dirpath, "audit-output-saved-state",
".json")
file_pathname = os.path.join(dirpath, filename)
file = open(file_pathname, "r")
e.saved_state = json.load(file)
def read_reported_outcomes(e):
election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
specification_pathname = os.path.join(election_pathname, "2-reported")
fieldnames = ["Contest", "Winner(s)"]
filename = utils.greatest_name(specification_pathname,
"23-reported-outcomes", ".csv")
file_pathname = os.path.join(specification_pathname, filename)
rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True)
for row in rows:
cid = row["Contest"]
winners = row["Winner(s)"]
utils.nested_set(e.ro_c, [cid], winners)
def read_audit_spec_collection(e, args):
""" Read 3-audit/31-audit-spec/audit-spec-collection.csv """
election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
audit_spec_pathname = os.path.join(election_pathname, "3-audit",
"31-audit-spec")
filename = utils.greatest_name(audit_spec_pathname,
"audit-spec-collection", ".csv")
file_pathname = os.path.join(audit_spec_pathname, filename)
fieldnames = ["Collection", "Max audit rate"]
rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False)
for row in rows:
pbcid = row["Collection"]
e.max_audit_rate_p[pbcid] = int(row["Max audit rate"])
def read_syn2_csv(e, synpar):
"""
Read file defining syn2 synthetic election spec.
"""
syn2_pathname = os.path.join(multi.ELECTIONS_ROOT, "syn2_specs")
filename = utils.greatest_name(syn2_pathname, synpar.election_dirname,
".csv")
file_pathname = os.path.join(syn2_pathname, filename)
fieldnames = [
"Contest", "Collection", "Reported Vote", "Actual Vote", "Number"
]
rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False)
return [(row["Contest"], row["Collection"], (row["Reported Vote"], ),
(row["Actual Vote"], ), row["Number"]) for row in rows]
def read_audit_spec_global(e, args):
""" Read 3-audit/31-audit-spec/audit-spec-global.csv """
election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
audit_spec_pathname = os.path.join(election_pathname, "3-audit",
"31-audit-spec")
filename = utils.greatest_name(audit_spec_pathname, "audit-spec-global",
".csv")
file_pathname = os.path.join(audit_spec_pathname, filename)
fieldnames = ["Global Audit Parameter", "Value"]
rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False)
for row in rows:
parameter = row["Global Audit Parameter"]
value = row["Value"]
if parameter == "Max audit stage time":
e.max_stage_time = value
def counting(prefix, dirpath="./elections"):
results = dd(int)
output_prefix = "audit-output-contest-status"
output_postfix = ".csv"
output_inner_dir = path.join("3-audit", "34-audit-output")
output_fields = [
"Measurement id", "Contest", "Risk Measurement Method", "Risk Limit",
"Risk Upset Threshold", "Sampling Mode", "Status", "Param 1", "Param 2"
]
for election_dir in os.listdir(dirpath):
# Find the correct directory
if election_dir.startswith(prefix):
election_dir = path.join(dirpath, election_dir)
output_dir = path.join(election_dir, output_inner_dir)
if path.exists(output_dir):
file = greatest_name(output_dir, output_prefix, output_postfix)
file = path.join(output_dir, file)
field_dict = read_csv_file(file,
required_fieldnames=output_fields)
print(file)
print(field_dict)
for row in field_dict:
results[row["Status"]] += 1
result_path = path.join(".", "results")
if not path.exists(result_path):
os.makedirs(result_path)
all_status = ["Upset", "Passed", "Open"]
result_csv = path.join(result_path, prefix + ".csv")
try:
with open(result_csv, 'w') as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=all_status)
writer.writeheader()
for data in [
results,
]:
writer.writerow(data)
except IOError:
print("I/O error")
return results
def read_audit_spec_seed(e, args):
"""
Read audit seed from 3-audit/31-audit-spec/audit-spec-seed.csv
Do not overwrite e.audit_seed if it was non-None
because this means it was already set from the command line.
"""
election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
audit_spec_pathname = os.path.join(election_pathname, "3-audit",
"31-audit-spec")
filename = utils.greatest_name(audit_spec_pathname, "audit-spec-seed",
".csv")
file_pathname = os.path.join(audit_spec_pathname, filename)
fieldnames = ["Audit seed"]
rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False)
for row in rows:
new_audit_seed = row["Audit seed"]
if e.audit_seed == None:
set_audit_seed(e, new_audit_seed)
def read_election_spec_contest_groups(e):
"""
Read file election-spec-contest-groups.csv, put results into Election e.
"""
election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
spec_pathname = os.path.join(election_pathname, "1-election-spec")
filename = utils.greatest_name(spec_pathname,
"election-spec-contest-groups", ".csv")
file_pathname = os.path.join(spec_pathname, filename)
fieldnames = ["Contest group", "Contest(s) or group(s)"]
rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True)
for row in rows:
gid = row["Contest group"]
if gid in e.cids:
raise ValueError(
"Contest group id {} must not also be a contest id.".format(
gid))
e.gids.append(gid)
e.cgids_g[gid] = row["Contest(s) or group(s)"]
def read_meta_csv(e, synpar):
"""
Read file defining syn3 synthetic election spec.
"""
syn3_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT, "syn3_specs",
synpar.election_dirname)
try:
# Use config dir if properly defined
if synpar.config_dirname is not None:
syn3_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT,
"syn3_specs", synpar.config_dirname)
except NameError:
pass
filename = utils.greatest_name(syn3_pathname, "meta", ".csv")
file_pathname = os.path.join(syn3_pathname, filename)
fieldnames = [
"Contest", "Collection", "Winner", "Audit Rate", "Contestants"
]
rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True)
return [(row["Contest"], row["Collection"], (row["Winner"], ),
row["Audit Rate"], row["Contestants"]) for row in rows]
def read_election_spec_general(e, election_dirname):
"""
Read file 1-election-spec/election-spec-general.csv, put results into Election e.
election_dirname is the name of the directory for the election
(e.g. "CO-2017-11") with ELECTIONS_ROOT
"""
election_pathname = os.path.join(multi.ELECTIONS_ROOT, election_dirname)
spec_pathname = os.path.join(election_pathname, "1-election-spec")
filename = utils.greatest_name(spec_pathname, "election-spec-general",
".csv")
file_pathname = os.path.join(spec_pathname, filename)
fieldnames = ["Attribute", "Value"]
rows = csv_readers.read_csv_file(file_pathname, fieldnames)
for row in rows:
if "Election name" == row["Attribute"]:
e.election_name = row["Value"]
elif "Election dirname" == row["Attribute"]:
new_election_dirname = row["Value"]
if new_election_dirname != election_dirname:
utils.mywarning(
("Inconsistent election_dirname {}"
"ignored in file {}.").format(new_election_dirname,
file_pathname))
else:
pass # everything OK
elif "Election date" == row["Attribute"]:
e.election_date = row["Value"]
elif "Election URL" == row["Attribute"]:
e.election_url = row["Value"]
for attribute in [
"election_name", "election_dirname", "election_date",
"election_url"
]:
if attribute not in vars(e):
utils.mywarning(
"Attribute {} not present in 11-general.csv.".format(
attribute))
if utils.warnings_given > 0:
utils.myerror("Too many errors; terminating.")
def read_audited_votes(e):
"""
Read audited votes from 3-audit/33-audited-votes/audited-votes-PBCID.csv
"""
election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
audited_votes_pathname = os.path.join(election_pathname, "3-audit",
"33-audited-votes")
for pbcid in e.pbcids:
safe_pbcid = ids.filename_safe(pbcid)
filename = utils.greatest_name(audited_votes_pathname,
"audited-votes-" + safe_pbcid, ".csv")
file_pathname = os.path.join(audited_votes_pathname, filename)
fieldnames = ["Collection", "Ballot id", "Contest", "Selections"]
rows = csv_readers.read_csv_file(file_pathname,
fieldnames,
varlen=True)
for row in rows:
pbcid = row["Collection"]
bid = row["Ballot id"]
cid = row["Contest"]
vote = row["Selections"]
utils.nested_set(e.av_cpb, [cid, pbcid, bid], vote)
def read_vote_csv(e, synpar, reported=False):
"""
Read file defining syn3 synthetic election spec.
"""
syn3_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT, "syn3_specs",
synpar.election_dirname)
try:
# Use config dir if properly defined
if synpar.config_dirname is not None:
syn3_pathname = os.path.join(OpenAuditTool.ELECTIONS_ROOT,
"syn3_specs", synpar.config_dirname)
except NameError:
pass
filename = utils.greatest_name(syn3_pathname,
"reported" if reported else "actual",
".csv")
file_pathname = os.path.join(syn3_pathname, filename)
fieldnames = ["Contest", "Collection", "Number", "Votes"]
rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=True)
return [(row["Contest"], row["Collection"], row["Number"], row["Votes"])
for row in rows]
def read_election_spec_collections(e):
"""
Read file 14-collections.csv, put results into Election e.
"""
election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
spec_pathname = os.path.join(election_pathname, "1-election-spec")
filename = utils.greatest_name(spec_pathname, "election-spec-collections",
".csv")
file_pathname = os.path.join(spec_pathname, filename)
fieldnames = [
"Collection", "Manager", "CVR type", "Required Contests",
"Possible Contests"
]
rows = csv_readers.read_csv_file(file_pathname, fieldnames, varlen=False)
for row in rows:
pbcid = row["Collection"]
e.pbcids.append(pbcid)
e.manager_p[pbcid] = row["Manager"]
e.cvr_type_p[pbcid] = row["CVR type"]
e.required_gid_p[pbcid] = row["Required Contests"]
e.possible_gid_p[pbcid] = row["Possible Contests"]
