def test_gmpy2_parallelism_is_safe(self):
cpus = cpu_count()
problem_size = 1000
secret_keys = Nonces(int_to_q_unchecked(3))[
0:problem_size] # list of 1000 might-as-well-be-random Q's
log_info(
f"testing GMPY2 powmod parallelism safety (cpus = {cpus}, problem_size = {problem_size})"
)
# compute in parallel
start = timer()
p = Pool(cpus)
keypairs = p.map(elgamal_keypair_from_secret, secret_keys)
end1 = timer()
# verify scalar
for keypair in keypairs:
self.assertEqual(
keypair.public_key,
elgamal_keypair_from_secret(keypair.secret_key).public_key,
)
end2 = timer()
p.close(
) # apparently necessary to avoid warnings from the Pool system
log_info(f"Parallelism speedup: {(end2 - end1) / (end1 - start):.3f}")
def elements_mod_q(draw: _DrawType):
"""
Generates an arbitrary element from [0,Q).
:param draw: Hidden argument, used by Hypothesis.
"""
return int_to_q_unchecked(draw(integers(min_value=0, max_value=Q - 1)))
def test_gmpy2_parallelism_is_safe(self):
"""
Ensures running lots of parallel exponentiations still yields the correct answer.
This verifies that nothing incorrect is happening in the GMPY2 library
"""
# Arrange
scheduler = Scheduler()
problem_size = 1000
random_secret_keys = Nonces(int_to_q_unchecked(3))[0:problem_size]
log_info(
f"testing GMPY2 powmod parallelism safety (cpus = {scheduler.cpu_count}, problem_size = {problem_size})"
)
# Act
start = timer()
keypairs = scheduler.schedule(
elgamal_keypair_from_secret,
[list([secret_key]) for secret_key in random_secret_keys],
)
end1 = timer()
# Assert
for keypair in keypairs:
self.assertEqual(
keypair.public_key,
elgamal_keypair_from_secret(keypair.secret_key).public_key,
)
end2 = timer()
scheduler.close()
log_info(f"Parallelism speedup: {(end2 - end1) / (end1 - start):.3f}")
def test_publish_private_data(self) -> None:
# Arrange
plaintext_ballots = [PlaintextBallot("", "", [])]
encrypted_ballots = [
make_ciphertext_ballot("", "", int_to_q_unchecked(0),
int_to_q_unchecked(0), [])
]
guardians = [Guardian("", 1, 1, 1)]
# Act
publish_private_data(
plaintext_ballots,
encrypted_ballots,
guardians,
)
# Assert
self.assertTrue(path.exists(RESULTS_DIR))
# Cleanup
rmtree(RESULTS_DIR)
def set_deserializers():
electionguard.serializable.set_deserializer(
lambda p, cls, **_: int_to_p_unchecked(maybe_base64_to_int(p)),
ElementModP # type: ignore
)
electionguard.serializable.set_deserializer(
lambda q, cls, **_: int_to_q_unchecked(maybe_base64_to_int(q)),
ElementModQ # type: ignore
)
electionguard.serializable.set_deserializer(
lambda i, cls, **_: maybe_base64_to_int(i),
int # type: ignore
)
def run_bench(filename: str, output_dir: Optional[str],
use_progressbar: bool) -> None:
start_time = timer()
print(f"Benchmarking: {filename}")
cvrs = read_dominion_csv(filename)
if cvrs is None:
print(f"Failed to read {filename}, terminating.")
exit(1)
rows, cols = cvrs.data.shape
parse_time = timer()
print(
f" Parse time: {parse_time - start_time: .3f} sec, {rows / (parse_time - start_time):.3f} ballots/sec"
)
assert rows > 0, "can't have zero ballots!"
# doesn't matter what the key is, so long as it's consistent for both runs
keypair = get_optional(
elgamal_keypair_from_secret(int_to_q_unchecked(31337)))
rtally_start = timer()
rtally = ray_tally_everything(
cvrs,
secret_key=keypair.secret_key,
verbose=True,
root_dir=output_dir,
use_progressbar=use_progressbar,
)
rtally_end = timer()
print(f"\nOVERALL PERFORMANCE")
print(f" Ray time: {rtally_end - rtally_start : .3f} sec")
print(
f" Ray rate: {rows / (rtally_end - rtally_start): .3f} ballots/sec"
)
if output_dir:
print(f"\nSANITY CHECK")
assert rtally.all_proofs_valid(
verbose=True,
recheck_ballots_and_tallies=False,
use_progressbar=use_progressbar,
), "proof failure!"
def _generate_encrypted_tally(
self,
metadata: InternalElectionDescription,
context: CiphertextElectionContext,
ballots: List[PlaintextBallot],
) -> CiphertextTally:
# encrypt each ballot
store = BallotStore()
for ballot in ballots:
encrypted_ballot = encrypt_ballot(ballot, metadata, context,
int_to_q_unchecked(1))
self.assertIsNotNone(encrypted_ballot)
# add to the ballot store
store.set(
encrypted_ballot.object_id,
from_ciphertext_ballot(encrypted_ballot, BallotBoxState.CAST),
)
tally = tally_ballots(store, metadata, context)
self.assertIsNotNone(tally)
return get_optional(tally)
def generate_election_keys(request: ElectionKeyPairRequest) -> ElectionKeyPair:
"""
Generate election key pairs for use in election process
:param request: Election key pair request
:return: Election key pair
"""
keys = generate_election_key_pair(
request.quorum,
int_to_q_unchecked(request.nonce)
if request.nonce is not None else None,
)
if not keys:
raise HTTPException(
status_code=500,
detail="Election keys failed to be generated",
)
return ElectionKeyPair(
public_key=str(keys.key_pair.public_key),
secret_key=str(keys.key_pair.secret_key),
proof=write_json_object(keys.proof),
polynomial=write_json_object(keys.polynomial),
)
def create_guardian(request: GuardianRequest) -> Guardian:
"""
Create a guardian for the election process with the associated keys
"""
election_keys = generate_election_key_pair(
request.quorum,
int_to_q_unchecked(request.nonce) if request.nonce is not None else None,
)
if request.auxiliary_key_pair is None:
auxiliary_keys = generate_rsa_auxiliary_key_pair()
else:
auxiliary_keys = request.auxiliary_key_pair
if not election_keys:
raise HTTPException(
status_code=500,
detail="Election keys failed to be generated",
)
if not auxiliary_keys:
raise HTTPException(
status_code=500, detail="Auxiliary keys failed to be generated"
)
return Guardian(
id=request.id,
sequence_order=request.sequence_order,
number_of_guardians=request.number_of_guardians,
quorum=request.quorum,
election_key_pair=ElectionKeyPair(
public_key=str(election_keys.key_pair.public_key),
secret_key=str(election_keys.key_pair.secret_key),
proof=write_json_object(election_keys.proof),
polynomial=write_json_object(election_keys.polynomial),
),
auxiliary_key_pair=AuxiliaryKeyPair(
public_key=auxiliary_keys.public_key, secret_key=auxiliary_keys.secret_key
),
)
def test_decrypt_contest_invalid_input_fails(
self,
contest_description: Tuple[str, ContestDescription],
keypair: ElGamalKeyPair,
nonce_seed: ElementModQ,
random_seed: int,
):
# Arrange
_, description = contest_description
random = Random(random_seed)
data = ballot_factory.get_random_contest_from(description, random)
placeholders = generate_placeholder_selections_from(
description, description.number_elected
)
description_with_placeholders = contest_description_with_placeholders_from(
description, placeholders
)
self.assertTrue(description_with_placeholders.is_valid())
# Act
subject = encrypt_contest(
data, description_with_placeholders, keypair.public_key, nonce_seed
)
self.assertIsNotNone(subject)
# tamper with the nonce
subject.nonce = int_to_q_unchecked(1)
result_from_nonce = decrypt_contest_with_nonce(
subject,
description_with_placeholders,
keypair.public_key,
remove_placeholders=False,
)
result_from_nonce_seed = decrypt_contest_with_nonce(
subject,
description_with_placeholders,
keypair.public_key,
nonce_seed,
remove_placeholders=False,
)
# Assert
self.assertIsNone(result_from_nonce)
self.assertIsNone(result_from_nonce_seed)
# Tamper with the encryption
subject.ballot_selections[0].message = ElGamalCiphertext(TWO_MOD_P, TWO_MOD_P)
result_from_key_tampered = decrypt_contest_with_secret(
subject,
description_with_placeholders,
keypair.public_key,
keypair.secret_key,
remove_placeholders=False,
)
result_from_nonce_tampered = decrypt_contest_with_nonce(
subject,
description_with_placeholders,
keypair.public_key,
remove_placeholders=False,
)
result_from_nonce_seed_tampered = decrypt_contest_with_nonce(
subject,
description_with_placeholders,
keypair.public_key,
nonce_seed,
remove_placeholders=False,
)
# Assert
self.assertIsNone(result_from_key_tampered)
self.assertIsNone(result_from_nonce_tampered)
self.assertIsNone(result_from_nonce_seed_tampered)
valid = proof.is_valid(ciphertext, keypair.public_key)
end2 = timer()
if not valid:
raise Exception(
"Wasn't expecting an invalid proof during a benchmark!")
return end1 - start1, end2 - end1
def identity(x: int) -> int:
"""Placeholder function used just to warm up the parallel mapper prior to benchmarking."""
return x
if __name__ == "__main__":
problem_sizes = (100, 500, 1000, 5000)
rands = Nonces(int_to_q_unchecked(31337))
speedup: Dict[int, float] = {}
print(f"CPUs detected: {cpu_count()}, launching thread pool")
pool = Pool(cpu_count())
# warm up the pool to help get consistent measurements
results = pool.map(identity, range(1, 30000))
assert results == list(range(1, 30000))
bench_start = timer()
for size in problem_sizes:
print("Benchmarking on problem size: ", size)
seeds = rands[0:size]
inputs = [
BenchInput(
def test_in_bounds_q_no_zero(self, q: ElementModQ):
self.assertTrue(q.is_in_bounds_no_zero())
self.assertFalse(ZERO_MOD_Q.is_in_bounds_no_zero())
self.assertFalse(int_to_q_unchecked(q.to_int() + Q).is_in_bounds_no_zero())
self.assertFalse(int_to_q_unchecked(q.to_int() - Q).is_in_bounds_no_zero())
def test_cached_one(self):
plaintext = int_to_q_unchecked(1)
ciphertext = g_pow_p(plaintext)
plaintext_again = discrete_log(ciphertext)
self.assertEqual(1, plaintext_again)
def test_nonces_type_errors(self):
n = Nonces(int_to_q_unchecked(3))
self.assertRaises(TypeError, len, n)
self.assertRaises(TypeError, lambda: n[1:])
self.assertRaises(TypeError, lambda: n.get_with_headers(-1))
def fast_tally_everything(
cvrs: DominionCSV,
pool: Optional[Pool] = None,
verbose: bool = True,
date: Optional[datetime] = None,
seed_hash: Optional[ElementModQ] = None,
master_nonce: Optional[ElementModQ] = None,
secret_key: Optional[ElementModQ] = None,
use_progressbar: bool = True,
) -> FastTallyEverythingResults:
"""
This top-level function takes a collection of Dominion CVRs and produces everything that
we might want for arlo-e2e: a list of encrypted ballots, their encrypted and decrypted tally,
and proofs of the correctness of the whole thing. The election `secret_key` is an optional
parameter. If absent, a random keypair is generated and used. Similarly, if a `seed_hash` or
`master_nonce` is not provided, random ones are generated and used.
For parallelism, a `multiprocessing.pool.Pool` may be provided, and should result in significant
speedups on multicore computers. If absent, the computation will proceed sequentially.
"""
rows, cols = cvrs.data.shape
if date is None:
date = datetime.now()
parse_time = timer()
log_and_print(f"Rows: {rows}, cols: {cols}", verbose)
ed, ballots, id_map = cvrs.to_election_description(date=date)
assert len(ballots) > 0, "can't have zero ballots!"
keypair = (elgamal_keypair_random() if secret_key is None else
elgamal_keypair_from_secret(secret_key))
assert keypair is not None, "unexpected failure with keypair computation"
secret_key, public_key = keypair
# This computation exists only to cause side-effects in the DLog engine, so the lame nonce is not an issue.
assert len(ballots) == get_optional(
elgamal_encrypt(m=len(ballots),
nonce=int_to_q_unchecked(3),
public_key=public_key)).decrypt(
secret_key), "got wrong ElGamal decryption!"
dlog_prime_time = timer()
log_and_print(
f"DLog prime time (n={len(ballots)}): {dlog_prime_time - parse_time: .3f} sec",
verbose,
)
cec = make_ciphertext_election_context(
number_of_guardians=1,
quorum=1,
elgamal_public_key=public_key,
description_hash=ed.crypto_hash(),
)
ied = InternalElectionDescription(ed)
# REVIEW THIS: is this cryptographically sound? Is the seed_hash properly a secret? Should
# it go in the output? The nonces are clearly secret. If you know them, you can decrypt.
if seed_hash is None:
seed_hash = rand_q()
if master_nonce is None:
master_nonce = rand_q()
nonces: List[ElementModQ] = Nonces(master_nonce)[0:len(ballots)]
# even if verbose is false, we still want to see the progress bar for the encryption
cballots = fast_encrypt_ballots(ballots,
ied,
cec,
seed_hash,
nonces,
pool,
use_progressbar=use_progressbar)
eg_encrypt_time = timer()
log_and_print(
f"Encryption time: {eg_encrypt_time - dlog_prime_time: .3f} sec",
verbose)
log_and_print(
f"Encryption rate: {rows / (eg_encrypt_time - dlog_prime_time): .3f} ballot/sec",
verbose,
)
tally: TALLY_TYPE = fast_tally_ballots(cballots, pool)
eg_tabulate_time = timer()
log_and_print(
f"Tabulation time: {eg_tabulate_time - eg_encrypt_time: .3f} sec",
verbose)
log_and_print(
f"Tabulation rate: {rows / (eg_tabulate_time - eg_encrypt_time): .3f} ballot/sec",
verbose,
)
log_and_print(
f"Encryption and tabulation: {rows} ballots / {eg_tabulate_time - dlog_prime_time: .3f} sec = {rows / (eg_tabulate_time - dlog_prime_time): .3f} ballot/sec",
verbose,
)
assert tally is not None, "tally failed!"
if verbose: # pragma: no cover
print("Decryption & Proofs: ")
decrypted_tally: DECRYPT_TALLY_OUTPUT_TYPE = fast_decrypt_tally(
tally, cec, keypair, seed_hash, pool, verbose)
eg_decryption_time = timer()
log_and_print(
f"Decryption time: {eg_decryption_time - eg_tabulate_time: .3f} sec",
verbose)
log_and_print(
f"Decryption rate: {len(decrypted_tally.keys()) / (eg_decryption_time - eg_tabulate_time): .3f} selection/sec",
verbose,
)
# Sanity-checking logic: make sure we don't have any unexpected keys, and that the decrypted totals
# match up with the columns in the original plaintext data.
for obj_id in decrypted_tally.keys():
assert obj_id in id_map, "object_id in results that we don't know about!"
cvr_sum = int(cvrs.data[id_map[obj_id]].sum())
decryption, proof = decrypted_tally[obj_id]
assert cvr_sum == decryption, f"decryption failed for {obj_id}"
# Assemble the data structure that we're returning. Having nonces in the ciphertext makes these
# structures sensitive for writing out to disk, but otherwise they're ready to go.
reported_tally: Dict[str, SelectionInfo] = {
k: SelectionInfo(
object_id=k,
encrypted_tally=tally[k],
# we need to forcibly convert mpz to int here to make serialization work properly
decrypted_tally=int(decrypted_tally[k][0]),
proof=decrypted_tally[k][1],
)
for k in tally.keys()
}
# strips the ballots of their nonces, which is important because those could allow for decryption
accepted_ballots = [ciphertext_ballot_to_accepted(x) for x in cballots]
return FastTallyEverythingResults(
metadata=cvrs.metadata,
cvr_metadata=cvrs.dataframe_without_selections(),
election_description=ed,
encrypted_ballot_memos={
ballot.object_id: make_memo_value(ballot)
for ballot in accepted_ballots
},
tally=SelectionTally(reported_tally),
context=cec,
)
def run_bench(filename: str, pool: Pool, file_dir: Optional[str]) -> None:
start_time = timer()
print(f"Benchmarking: {filename}")
log_info(f"Benchmarking: {filename}")
cvrs = read_dominion_csv(filename)
if cvrs is None:
print(f"Failed to read {filename}, terminating.")
exit(1)
rows, cols = cvrs.data.shape
parse_time = timer()
print(f" Parse time: {parse_time - start_time: .3f} sec")
assert rows > 0, "can't have zero ballots!"
# doesn't matter what the key is, so long as it's consistent for both runs
keypair = get_optional(elgamal_keypair_from_secret(int_to_q_unchecked(31337)))
tally_start = timer()
tally = fast_tally_everything(
cvrs, pool, verbose=True, secret_key=keypair.secret_key
)
if file_dir:
write_fast_tally(tally, file_dir + "_fast")
tally_end = timer()
assert tally.all_proofs_valid(verbose=True), "proof failure!"
print(f"\nstarting ray.io parallelism")
rtally_start = timer()
rtally = ray_tally_everything(
cvrs,
secret_key=keypair.secret_key,
root_dir=file_dir + "_ray" if file_dir else None,
)
rtally_end = timer()
if file_dir:
rtally_as_fast = rtally.to_fast_tally()
assert rtally_as_fast.all_proofs_valid(verbose=True), "proof failure!"
assert tally.equivalent(
rtally_as_fast, keypair, pool
), "tallies aren't equivalent!"
# Note: tally.equivalent() isn't quite as stringent as asserting that absolutely
# everything is identical, but it's a pretty good sanity check for our purposes.
# In tests/test_ray_tally.py, test_ray_and_multiprocessing_agree goes the extra
# distance to create identical tallies from each system and assert their equality.
print(f"\nOVERALL PERFORMANCE")
print(f" Pool time: {tally_end - tally_start: .3f} sec")
print(f" Pool rate: {rows / (tally_end - tally_start): .3f} ballots/sec")
print(f" Ray time: {rtally_end - rtally_start : .3f} sec")
print(f" Ray rate: {rows / (rtally_end - rtally_start): .3f} ballots/sec")
print(
f" Ray speedup: {(tally_end - tally_start) / (rtally_end - rtally_start) : .3f} (>1.0 = ray is faster, <1.0 = ray is slower)"
)
if file_dir is not None:
shutil.rmtree(file_dir + "_ray", ignore_errors=True)
shutil.rmtree(file_dir + "_fast", ignore_errors=True)
