def dispatch(e, args):
e.election_dirname = ids.filename_safe(args.election_dirname)
e.election_name = e.election_dirname
dirpath = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
if os.path.exists(dirpath):
utils.mywarning(
"Erasing previous contents of directory {}.".format(dirpath))
subdirs = ["1-election-spec", "2-reported", "3-audit"]
for subdir in subdirs:
dirpathx = os.path.join(dirpath, subdir)
if os.path.exists(dirpathx):
shutil.rmtree(dirpathx)
utils.mywarning("Directory {} erased.".format(dirpathx))
if args.syn_type == '1':
syn1.generate_syn_type_1(e, args)
elif args.syn_type == '2':
syn2.generate_syn_type_2(e, args)
else:
print("Illegal syn_type:", args.syn_type)
print(" Done. Synthetic election written to:", dirpath)
def check_audited_votes(e):
"""
old code; was in check_reported, but moved here temporarily
"""
if not isinstance(e.av_cpb, dict):
utils.myerror("e.av_cpb is not a dict.")
for cid in e.av_cpb:
if cid not in e.cids:
utils.mywarning("e.av_cpb key {} is not in e.cids.".format(cid))
for pbcid in e.av_cpb[cid]:
if pbcid not in e.pbcids:
utils.mywarning(
"e.av_cpb[{}] key `{}` is not in e.pbcids".format(
cid, pbcid))
if not isinstance(e.av_cpb[cid][pbcid], dict):
utils.myerror("e.av_cpb[{}][{}] is not a dict.".format(
cid, pbcid))
bidsset = set(e.bids_p[pbcid])
for bid in e.av_cpb[cid][pbcid]:
if bid not in bidsset:
utils.mywarning(
"bid `{}` from e.av_cpb[{}][{}] is not in e.bids_p[{}]."
.format(bid, cid, pbcid, pbcid))
for cid in e.cids:
if cid not in e.av_cpb:
utils.mywarning("cid `{}` is not a key for e.av_cpb.".format(cid))
for pbcid in e.possible_pbcid_c[cid]:
if pbcid not in e.av_cpb[cid]:
utils.mywarning("pbcid `{}` is not in e.av_cpb[{}].".format(
pbcid, cid))
def check_id(id, check_for_whitespace=False):
if not isinstance(id, str) or not id.isprintable():
utils.mywarning("id is not string or is not printable: {}".format(id))
if check_for_whitespace:
for c in id:
if c.isspace():
utils.mywarning("id `id` contains whitespace.")
break
def read_reported_ballot_manifests(e):
"""
Read ballot manifest file 21-reported-ballot-manifests and expand rows if needed.
"""
election_pathname = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
specification_pathname = os.path.join(election_pathname, "2-reported",
"21-reported-ballot-manifests")
fieldnames = [
"Collection", "Box", "Position", "Stamp", "Ballot id",
"Number of ballots", "Required Contests", "Possible Contests",
"Comments"
]
for pbcid in e.pbcids:
safe_pbcid = ids.filename_safe(pbcid)
filename = utils.greatest_name(specification_pathname,
"manifest-" + safe_pbcid, ".csv")
file_pathname = os.path.join(specification_pathname, filename)
rows = csv_readers.read_csv_file(file_pathname,
fieldnames,
varlen=False)
for row in rows:
pbcid = row["Collection"]
boxid = row["Box"]
position = row["Position"]
stamp = row["Stamp"]
bid = row["Ballot id"]
try:
num = int(row["Number of ballots"])
except ValueError:
utils.myerror(
"Number {} of ballots not an integer.".format(num))
if num <= 0:
utils.mywarning(
"Number {} of ballots not positive.".format(num))
req = row["Required Contests"]
poss = row["Possible Contests"]
comments = row["Comments"]
bids = utils.count_on(bid, num)
stamps = utils.count_on(stamp, num)
positions = utils.count_on(position, num)
for i in range(num):
# utils.nested_set(e.bids_p, [pbcid, bids[i]], True)
if pbcid not in e.bids_p:
e.bids_p[pbcid] = []
e.bids_p[pbcid].append(bids[i])
utils.nested_set(e.boxid_pb, [pbcid, bids[i]], boxid)
utils.nested_set(e.position_pb, [pbcid, bids[i]], position[i])
utils.nested_set(e.stamp_pb, [pbcid, bids[i]], stamps[i])
utils.nested_set(e.required_gid_pb, [pbcid, bids[i]], req)
utils.nested_set(e.possible_gid_pb, [pbcid, bids[i]], poss)
utils.nested_set(e.comments_pb, [pbcid, bids[i]], comments)
def check_audit_spec(e):
if not isinstance(e.risk_limit_m, dict):
utils.myerror("e.risk_limit_m is not a dict.")
for mid in e.risk_limit_m:
if mid not in e.mids:
utils.mywarning(
"e.risk_limit_m mid key `{}` is not in e.mids.".format(mid))
if not (0.0 <= float(e.risk_limit_m[mid]) <= 1.0):
utils.mywarning(
"e.risk_limit_m[{}] not in interval [0,1]".format(mid))
if not isinstance(e.max_audit_rate_p, dict):
utils.myerror("e.max_audit_rate_p is not a dict.")
for pbcid in e.max_audit_rate_p:
if pbcid not in e.pbcids:
utils.mywarning(
"pbcid `{}` is a key for e.max_audit_rate_p but not in e.pbcids."
.format(pbcid))
if not 0 <= int(e.max_audit_rate_p[pbcid]):
utils.mywarning(
"e.max_audit_rate_p[{}] must be nonnegative.".format(pbcid))
if utils.warnings_given > 0:
utils.myerror("Too many errors; terminating.")
def process_args(e, args):
e.election_dirname = ids.filename_safe(args.election_dirname)
e.election_name = e.election_dirname
dirpath = os.path.join(multi.ELECTIONS_ROOT, e.election_dirname)
if os.path.exists(dirpath):
utils.mywarning("Existing directory {} erased.".format(dirpath))
shutil.rmtree(dirpath)
if args.syn_type == '1':
syn1.generate_syn_type_1(e, args)
elif args.syn_type == '2':
syn2.generate_syn_type_2(e, args)
else:
print("Illegal syn_type:", args.syn_type)
print(" Done. Synthetic election written to:", dirpath)
def reachable_from(e, gid, gids, cids, stack):
"""
Find all gids and cids reachable from initial
gid in 0 or more steps.
This works even if the graph contains cycles.
"""
if gid in gids:
if gid in stack:
utils.mywarning("Group id {} is in a cycle!".format(gid))
return
gids.add(gid)
for cgid in e.cgids_g[gid]:
if cgid in e.cids:
cids.append(cgid)
else:
stack.append(gid)
reachable_from(e, cgid, gids, cids, stack)
stack.pop()
def read_election_spec_general(e, election_dirname):
"""
Read file 1-election-spec/election-spec-general.csv, put results into Election e.
election_dirname is the name of the directory for the election
(e.g. "CO-2017-11") with ELECTIONS_ROOT
"""
election_pathname = os.path.join(multi.ELECTIONS_ROOT, election_dirname)
spec_pathname = os.path.join(election_pathname, "1-election-spec")
filename = utils.greatest_name(spec_pathname, "election-spec-general",
".csv")
file_pathname = os.path.join(spec_pathname, filename)
fieldnames = ["Attribute", "Value"]
rows = csv_readers.read_csv_file(file_pathname, fieldnames)
for row in rows:
if "Election name" == row["Attribute"]:
e.election_name = row["Value"]
elif "Election dirname" == row["Attribute"]:
new_election_dirname = row["Value"]
if new_election_dirname != election_dirname:
utils.mywarning(
("Inconsistent election_dirname {}"
"ignored in file {}.").format(new_election_dirname,
file_pathname))
else:
pass # everything OK
elif "Election date" == row["Attribute"]:
e.election_date = row["Value"]
elif "Election URL" == row["Attribute"]:
e.election_url = row["Value"]
for attribute in [
"election_name", "election_dirname", "election_date",
"election_url"
]:
if attribute not in vars(e):
utils.mywarning(
"Attribute {} not present in 11-general.csv.".format(
attribute))
if utils.warnings_given > 0:
utils.myerror("Too many errors; terminating.")
def reachable_from(e, gid, gids, cids, stack):
"""
Find all gids and cids reachable from initial
gid in 0 or more steps. Main output of interest
is the input list "cids", which has all reachable
contests appended to it.
This works even if the graph contains cycles.
Algorithm is depth-first search (DFS).
"""
if gid in gids:
if gid in stack:
utils.mywarning("Group id {} is in a cycle!".format(gid))
return
gids.add(gid)
for cgid in e.cgids_g[gid]:
# Note: 'cgid' means 'contest or group id'
if cgid in e.cids:
cids.append(cgid)
else:
stack.append(gid)
reachable_from(e, cgid, gids, cids, stack)
stack.pop()
def check_election_spec(e):
if not isinstance(e.cids, (list, tuple, set)):
utils.myerror("e.cids is not a set.")
if len(e.cids) == 0:
utils.myerror("e.cids is an empty list of contests.")
for cid in e.cids:
check_id(cid)
if not isinstance(e.pbcids, (list, tuple)):
utils.myerror("e.pbcids is not a list or a tuple.")
if len(e.pbcids) == 0:
utils.myerror("e.pbcids is an empty list of pbcids.")
for pbcid in e.pbcids:
check_id(pbcid)
for cid in e.selids_c:
if cid not in e.cids:
utils.myerror(
"e.selids_c has a key `{}` not in e.cids.".format(cid))
for selid in e.selids_c[cid]:
check_id(selid)
for cid in e.cids:
if cid not in e.selids_c:
utils.mywarning("cid `{}` should be key in e.selids_c".format(cid))
if not isinstance(e.cvr_type_p, dict):
utils.myerror("e_cvr_type is not a dict.")
for pbcid in e.cvr_type_p:
if pbcid not in e.pbcids:
utils.mywarning("pbcid `{}` is not in e.pbcids".format(pbcid))
if e.cvr_type_p[pbcid] not in ["CVR", "noCVR"]:
utils.mywarning(
"e.cvr_type_p[{}]==`{}` is not CVR or noCVR".format(
pbcid, e.cvr_type_p[pbcid]))
for pbcid in e.pbcids:
if pbcid not in e.cvr_type_p:
utils.mywarning(
"pbcid `{}` not key in e.cvr_type_p.".format(pbcid))
if utils.warnings_given > 0:
utils.myerror(
"election_spec.check_election_spec: Too many errors or warnings; terminating."
)
def check_reported(e):
if not isinstance(e.rn_cpr, dict):
utils.myerror("e.rn_cpr is not a dict.")
for cid in e.rn_cpr:
if cid not in e.cids:
utils.mywarning("cid `{}` not in e.cids.".format(cid))
for pbcid in e.rn_cpr[cid]:
if pbcid not in e.pbcids:
utils.mywarning("pbcid `{}` is not in e.pbcids.".format(pbcid))
for cid in e.rn_cpr:
for pbcid in e.rn_cpr[cid]:
for rv in e.rn_cpr[cid][pbcid]:
for selid in rv:
if selid not in e.selids_c[cid] and selid[0].isalnum():
utils.mywarning(
"selid `{}` is not in e.selids_c[{}].".format(
selid, cid))
for cid in e.rn_cpr:
for pbcid in e.rn_cpr[cid]:
for rv in e.rn_cpr[cid][pbcid]:
if not isinstance(e.rn_cpr[cid][pbcid][rv], int):
utils.mywarning(
"value `e.rn_cpr[{}][{}][{}] = `{}` is not an integer."
.format(cid, pbcid, rv, e.rn_cpr[cid][pbcid][rv]))
if not (0 <= e.rn_cpr[cid][pbcid][rv] <= e.rn_p[pbcid]):
utils.mywarning(
"value `e.rn_cpr[{}][{}][{}] = `{}` is out of range 0:{}."
.format(cid, pbcid, rv, e.rn_cpr[cid][pbcid][rv],
e.rn_p[pbcid]))
if not (0 <= e.rn_cpr[cid][pbcid][rv] <= e.rn_c[cid]):
utils.mywarning(
"value `e.rn_cpr[{}][{}][{}] = `{}` is out of range 0:{}."
.format(cid, pbcid, rv, e.rn_cpr[cid][pbcid][rv],
e.rn_p[pbcid]))
for cid in e.cids:
for rv in e.votes_c[cid]:
if e.rn_cr[cid][rv] != \
sum([e.rn_cpr[cid][pbcid][rv] for pbcid in e.rn_cpr[cid]]):
utils.mywarning(
"sum of e.rn_cpr[{}][*][{}] is not e.rn_cr[{}][{}].".
format(cid, rv, cid, rv))
for cid in e.cids:
if cid not in e.rn_cpr:
utils.mywarning("cid `{}` is not a key for e.rn_cpr".format(cid))
for pbcid in e.possible_pbcid_c[cid]:
if pbcid not in e.rn_cpr[cid]:
utils.mywarning(
"pbcid {} is not a key for e.rn_cpr[{}].".format(
pbcid, cid))
# for selid in e.selids_c[cid]:
# assert selid in e.rn_cpr[cid][pbcid], (cid, pbcid, selid)
# ## not necessary, since missing selids have assumed count of 0
if not isinstance(e.rn_c, dict):
utils.myerror("e.rn_c is not a dict.")
for cid in e.rn_c:
if cid not in e.cids:
utils.mywarning("e.rn_c key `{}` is not in e.cids.".format(cid))
if not isinstance(e.rn_c[cid], int):
utils.mywarning("e.rn_c[{}] = {} is not an integer.".format(
cid, e.rn_c[cid]))
for cid in e.cids:
if cid not in e.rn_c:
utils.mywarning("cid `{}` is not a key for e.rn_c".format(cid))
if not isinstance(e.rn_cr, dict):
utils.myerror("e.rn_cr is not a dict.")
for cid in e.rn_cr:
if cid not in e.cids:
utils.mywarning(
"e.rn_cr key cid `{}` is not in e.cids".format(cid))
for vote in e.rn_cr[cid]:
for selid in vote:
if (not ids.is_writein(selid) and not ids.is_error_selid(selid)) \
and not selid in e.selids_c[cid]:
utils.mywarning(
"e.rn_cr[{}] key `{}` is not in e.selids_c[{}]".format(
cid, selid, cid))
if not isinstance(e.rn_cr[cid][vote], int):
utils.mywarning(
"e.rn_cr[{}][{}] = {} is not an integer.".format(
cid, vote, e.rn_cr[cid][vote]))
for cid in e.cids:
if cid not in e.rn_cr:
utils.mywarning("cid `{}` is not a key for e.rn_cr".format(cid))
if not isinstance(e.bids_p, dict):
utils.myerror("e.bids_p is not a dict.")
for pbcid in e.pbcids:
# if not isinstance(e.bids_p[pbcid], dict):
# utils.myerror("e.bids_p[{}] is not a dict.".format(pbcid))
if not isinstance(e.bids_p[pbcid], list):
utils.myerror("e.bids_p[{}] is not a list.".format(pbcid))
if not isinstance(e.rv_cpb, dict):
utils.myerror("e.rv_cpb is not a dict.")
for cid in e.rv_cpb:
if cid not in e.cids:
utils.mywarning("e.rv_cpb key `{}` is not in e.cids.".format(cid))
for pbcid in e.rv_cpb[cid]:
if pbcid not in e.pbcids:
utils.mywarning(
"e.rv_cpb[{}] key `{}` is not in e.pbcids.".format(
cid, pbcid))
if not isinstance(e.rv_cpb[cid][pbcid], dict):
utils.myerror("e.rv_cpb[{}][{}] is not a dict.".format(
cid, pbcid))
bidsset = set(e.bids_p[pbcid])
for bid in e.rv_cpb[cid][pbcid]:
if bid not in bidsset:
utils.mywarning(
"bid `{}` from e.rv_cpb[{}][{}] is not in e.bids_p[{}]."
.format(bid, cid, pbcid, pbcid))
for cid in e.cids:
if cid not in e.rv_cpb:
utils.mywarning("cid `{}` is not a key in e.rv_cpb.".format(cid))
for pbcid in e.possible_pbcid_c[cid]:
if pbcid not in e.rv_cpb[cid]:
utils.mywarning(
("pbcid `{}` from e.possible_pbcid_c[{}] "
"is not a key for e.rv_cpb[{}].").format(pbcid, cid, cid))
if not isinstance(e.ro_c, dict):
utils.myerror("e.ro_c is not a dict.")
for cid in e.ro_c:
if cid not in e.cids:
utils.mywarning(
"cid `{}` from e.rv_cpb is not in e.cids".format(cid))
for cid in e.cids:
if cid not in e.ro_c:
utils.mywarning("cid `{}` is not a key for e.ro_c.".format(cid))
if utils.warnings_given > 0:
utils.myerror("Too many errors; terminating.")
def read_csv_file(filename, required_fieldnames=None, varlen=False):
"""
Read CSV file and check required fieldnames present; varlen if variable-length rows.
"""
# print("Reading CSV file:", filename)
with open(filename) as file:
reader = csv.reader(file)
rows = [row for row in reader]
fieldnames = rows[0]
rows = rows[1:]
# gather, clean, and trim field names, eliminating blanks
fieldnames = [ids.clean_id(fieldname) for fieldname in fieldnames]
while len(fieldnames) > 0 and fieldnames[-1] == '':
fieldnames.pop()
if len(set(fieldnames)) != len(fieldnames):
utils.myerror("Duplicate field name:" + fieldnames)
# data rows
row_dicts = []
for row in rows:
row = ["" if item == None else ids.clean_id(item) for item in row]
while len(row) > 0 and row[-1] == '':
row.pop()
if not varlen:
if len(row) > len(fieldnames):
utils.mywarning("Ignoring extra values in row:" + str(row))
row = row[:len(fieldnames)]
while len(row) < len(fieldnames):
row.append("")
row_dict = {}
for (fieldname, value) in zip(fieldnames, row):
row_dict[fieldname] = value
if varlen:
if len(row) < len(fieldnames) - 1:
if len(row) > 0:
utils.mywarning("Ignoring too-short row:" + str(row))
continue
last_fieldname = fieldnames[-1]
last_value = tuple(row[len(fieldnames) - 1:])
row_dict[last_fieldname] = last_value
row_dicts.append(row_dict)
if required_fieldnames != None:
# check that all required fieldnames are present
required_fieldnames = [
ids.clean_id(id) for id in required_fieldnames
]
missing_fieldnames = set(required_fieldnames).difference(
set(fieldnames))
if len(missing_fieldnames) > 0:
utils.myerror(
"File {} has fieldnames {}, while {} are required. Missing {}."
.format(filename, fieldnames, required_fieldnames,
missing_fieldnames))
# check to see if extra fieldnames present; warn user if so
extra_fieldnames = set(fieldnames).difference(
set(required_fieldnames))
if len(extra_fieldnames) > 0:
utils.mywarning(
"File {} has extra fieldnames (ignored): {}".format(
filename, extra_fieldnames))
return row_dicts