def trustee_upload_decryption_factor(request, election, trustee_email):
keyshare = KeyShare.objects.get(election=election, email=trustee_email)
if keyshare.password != request.POST['password']:
return "failure: password doesn't match"
keyshare.decryption_factors = utils.from_json(request.POST['decryption_factors'])
keyshare.decryption_proofs = utils.from_json(request.POST['decryption_proofs'])
keyshare.save()
return SUCCESS
def set_cover(output_root, seeds):
print("Reading input file")
guide_to_seqnames = from_json(
paths["guide_to_seqnames"].format(output_root))
guides_to_keep = from_json(paths["guides_to_keep"].format(output_root))
print("Converting guide2seqnames to use sets")
new_guide_to_seqnames = {}
for guide in guides_to_keep:
new_guide_to_seqnames[guide] = set(guide_to_seqnames[guide])
guide_to_seqnames = new_guide_to_seqnames
print("Getting all_sequences")
all_sequences = set()
for _, seqnames in guide_to_seqnames.items():
all_sequences.update(seqnames)
print("Creating minimal guide set")
covered = set()
data = []
for guide in seeds:
covered.update(guide_to_seqnames[guide])
data.append({
"guide": guide,
"cumulative_sequences_covered": len(covered)
})
print(guide)
print("Num covered: {} out of {}".format(len(covered),
len(all_sequences)))
while len(covered) < len(all_sequences):
# Make sure that guide_to_seqnames is up to date
guide_to_seqnames = remove_covered_from_guides(guide_to_seqnames,
covered)
# Add one more guide to the minipool
guide = diverse_get_guide(guide_to_seqnames)
covered.update(guide_to_seqnames[guide])
data.append({
"guide": guide,
"cumulative_sequences_covered": len(covered)
})
print(guide)
print("Num covered: {} out of {}".format(len(covered),
len(all_sequences)))
to_json(paths["final_guides"].format(output_root), data)
guide_to_seqnames = from_json(
paths["guide_to_seqnames"].format(output_root))
guide_to_seqnames = {
dat["guide"]: guide_to_seqnames[dat["guide"]]
for dat in data
}
to_json(paths["final_guide_to_seqnames"].format(output_root),
guide_to_seqnames)
def trustee_upload_decryption_factor(request, election, trustee_email):
keyshare = KeyShare.objects.get(election=election, email=trustee_email)
if keyshare.password != request.POST['password']:
return "failure: password doesn't match"
keyshare.decryption_factors = utils.from_json(
request.POST['decryption_factors'])
keyshare.decryption_proofs = utils.from_json(
request.POST['decryption_proofs'])
keyshare.save()
return SUCCESS
def trustee_upload_pk(request, election, trustee_email):
keyshare = KeyShare.objects.get(election=election, email=trustee_email)
# is this authenticated properly, first api_client, otherwise password?
api_client = get_api_client(request)
if not api_client_can_admin_election(api_client, election):
if keyshare.password != request.POST['password']:
return HttpResponseServerError("failure: bad password")
election.public_key = None
election.save()
keyshare.public_key = algs.EGPublicKey.fromJSONDict(utils.from_json(request.POST['public_key']))
keyshare.pok = utils.from_json(request.POST['pok'])
keyshare.save()
return SUCCESS
def _do_tally(self, election_id):
# log back in as administrator
self.setup_login()
# encrypted tally
response = self.client.post("/helios/elections/%s/compute_tally" % election_id, {
"csrf_token" : self.client.session['csrf_token']
})
self.assertRedirects(response, "/helios/elections/%s/view" % election_id)
# should trigger helios decryption automatically
self.assertNotEquals(models.Election.objects.get(uuid=election_id).get_helios_trustee().decryption_proofs, None)
# combine decryptions
response = self.client.post("/helios/elections/%s/combine_decryptions" % election_id, {
"csrf_token" : self.client.session['csrf_token'],
})
# after tallying, we now are supposed to see the email screen
# with the right template value of 'result'
self.assertRedirects(response, "/helios/elections/%s/voters/email?template=result" % election_id)
# check that tally matches
response = self.client.get("/helios/elections/%s/result" % election_id)
self.assertEquals(utils.from_json(response.content), [[0,1]])
def set_encrypted_vote(self, votes_json_string):
# Check the proof on the vote
pk = self.election.public_key
election_obj = self.election.toElection()
vote_dict = utils.from_json(votes_json_string)
# verify
# turned off for now (Ben- 2008-11-28)
#enc_vote = electionalgs.EncryptedVote.fromJSONDict(vote_dict, pk)
#if not enc_vote.verify(election_obj):
# raise Exception("Vote does not verify")
# store this current vote in the voter structure
self.vote = vote_dict
if vote_dict:
self.vote_hash = self.compute_vote_hash()
else:
self.vote_hash = None
self.cast_id = str(datetime.datetime.utcnow()) + str(self.voter_id)
# store the vote
v = Vote.objects.create(cast_at = datetime.datetime.utcnow(), vote = vote_dict, vote_hash = self.vote_hash, voter= self)
self.save()
def set_encrypted_vote(self, votes_json_string):
# Check the proof on the vote
pk = self.election.public_key
election_obj = self.election.toElection()
vote_dict = utils.from_json(votes_json_string)
# verify
# turned off for now (Ben- 2008-11-28)
#enc_vote = electionalgs.EncryptedVote.fromJSONDict(vote_dict, pk)
#if not enc_vote.verify(election_obj):
# raise Exception("Vote does not verify")
# store this current vote in the voter structure
self.vote = vote_dict
if vote_dict:
self.vote_hash = self.compute_vote_hash()
else:
self.vote_hash = None
self.cast_id = str(datetime.datetime.utcnow()) + str(self.voter_id)
# store the vote
v = Vote.objects.create(cast_at=datetime.datetime.utcnow(),
vote=vote_dict,
vote_hash=self.vote_hash,
voter=self)
self.save()
def run(args):
tmp_file = args.path + '/' + args.file + '/' + args.job + '.' + args.tagOwner + '.'
real_path = args.path + "/" + args.file + "/" + args.tagOwner + "." + args.file + ".data"
if args.n != '':
return up(int(args.n), tmp_file, args)
try:
meta = get_meta(args.path, args.tagOwner, args.file)
# meta_json = utils.decode(args.meta)
meta_json = utils.decode(meta)
meta = utils.from_json(meta_json)
except:
return utils.log('err_parser_meta', args.file, 'error')
if not verify(meta, real_path):
return utils.log('err_verify_data', args.file, 'error')
num = int((meta['size'] - 1) / 3000000)
col_names = list(meta['colName'].split(','))
if not tmp(real_path, tmp_file, args.salt, col_names, args.encryptedColumn,
args.unencryptedColumn):
return utils.log('err_generate_encrypt', args.file, 'error')
up(num, tmp_file, args)
utils.log('info_impact', args.file)
def _do_tally(self, election_id):
# log back in as administrator
self.setup_login()
# encrypted tally
response = self.client.post(
"/helios/elections/%s/compute_tally" % election_id,
{"csrf_token": self.client.session['csrf_token']})
self.assertRedirects(response,
"/helios/elections/%s/view" % election_id)
# should trigger helios decryption automatically
self.assertNotEquals(
models.Election.objects.get(
uuid=election_id).get_helios_trustee().decryption_proofs, None)
# combine decryptions
response = self.client.post(
"/helios/elections/%s/combine_decryptions" % election_id, {
"csrf_token": self.client.session['csrf_token'],
})
# after tallying, we now are supposed to see the email screen
# with the right template value of 'result'
self.assertRedirects(
response,
"/helios/elections/%s/voters/email?template=result" % election_id)
# check that tally matches
response = self.client.get("/helios/elections/%s/result" % election_id)
self.assertEquals(utils.from_json(response.content), [[0, 1]])
def _do_tally(self, election_id):
# log back in as administrator
self.setup_login()
# encrypted tally
response = self.client.post("/helios/elections/%s/compute_tally" % election_id, {
"csrf_token" : self.client.session['csrf_token']
})
self.assertRedirects(response, "/helios/elections/%s/view" % election_id)
# should trigger helios decryption automatically
self.assertNotEquals(models.Election.objects.get(uuid=election_id).get_helios_trustee().decryption_proofs, None)
# combine decryptions
response = self.client.post("/helios/elections/%s/combine_decryptions" % election_id, {
"csrf_token" : self.client.session['csrf_token'],
})
# after tallying, we now go back to election_view
self.assertRedirects(response, "/helios/elections/%s/view" % election_id)
# check that we can't get the tally yet
response = self.client.get("/helios/elections/%s/result" % election_id)
self.assertEquals(response.status_code, 403)
# release
response = self.client.post("/helios/elections/%s/release_result" % election_id, {
"csrf_token" : self.client.session['csrf_token'],
})
# check that tally matches
response = self.client.get("/helios/elections/%s/result" % election_id)
self.assertEquals(utils.from_json(response.content), [[0,1]])
def _do_tally(self, election_id):
# log back in as administrator
self.setup_login()
# encrypted tally
response = self.client.post("/helios/elections/%s/compute_tally" % election_id, {
"csrf_token" : self.client.session['csrf_token']
})
self.assertRedirects(response, "/helios/elections/%s/view" % election_id)
# should trigger helios decryption automatically
self.assertNotEquals(models.Election.objects.get(uuid=election_id).get_helios_trustee().decryption_proofs, None)
# combine decryptions
response = self.client.post("/helios/elections/%s/combine_decryptions" % election_id, {
"csrf_token" : self.client.session['csrf_token'],
})
# after tallying, we now go back to election_view
self.assertRedirects(response, "/helios/elections/%s/view" % election_id)
# check that we can't get the tally yet
response = self.client.get("/helios/elections/%s/result" % election_id)
self.assertEquals(response.status_code, 403)
# release
response = self.client.post("/helios/elections/%s/release_result" % election_id, {
"csrf_token" : self.client.session['csrf_token'],
})
# check that tally matches
response = self.client.get("/helios/elections/%s/result" % election_id)
self.assertEquals(utils.from_json(response.content), [[0,1]])
def trustee_upload_pk(request, election, trustee_email):
keyshare = KeyShare.objects.get(election=election, email=trustee_email)
# is this authenticated properly, first api_client, otherwise password?
api_client = get_api_client(request)
if not api_client_can_admin_election(api_client, election):
if keyshare.password != request.POST['password']:
return HttpResponseServerError("failure: bad password")
election.public_key = None
election.save()
keyshare.public_key = algs.EGPublicKey.fromJSONDict(
utils.from_json(request.POST['public_key']))
keyshare.pok = utils.from_json(request.POST['pok'])
keyshare.save()
return SUCCESS
def one_election_set_pk(request, election):
if election.public_key:
return HttpResponseServerError("failure: Public Key exists already")
election.public_key = algs.EGPublicKey.fromJSONDict(utils.from_json(request.POST['public_key_json']))
election.save()
return SUCCESS
def get_cart():
url = reverse('api_dispatch_list', kwargs={
'resource_name': 'social_cart',
'api_name': 'v1'
})
result = world.client.get(url)
assert result.status_code == 200
return from_json(result.content)
def delete(self):
# time.sleep(5)
args = parser.parse_args()
flight = from_json(args["flight"])
if flight["id"] in flights.keys():
flights.pop(flight["id"])
self.save()
return flight #to_json(flight)
def one_election_set_pk(request, election):
if election.public_key:
return HttpResponseServerError("failure: Public Key exists already")
election.public_key = algs.EGPublicKey.fromJSONDict(
utils.from_json(request.POST['public_key_json']))
election.save()
return SUCCESS
def one_election_set_tally(request, election):
"""
Set the tally and proof.
"""
tally_obj = utils.from_json(request.POST['tally'])
election.set_result(tally_obj['result'], tally_obj['result_proof'])
election.save()
return SUCCESS
def get(self, path):
if path == ['~history']:
return self.history
src = redis.get(self.uri)
if src:
self.value = from_json(src)
return self
else:
return None
def one_election_set_tally(request, election): """ Set the tally and proof. """ tally_obj = utils.from_json(request.POST['tally']) election.set_result(tally_obj['result'], tally_obj['result_proof']) election.save() return SUCCESS
def get_accuracy(params):
gnn_graph = utils.build_gnn_graph(dataset, params)
model = GNN(gnn_graph).to(device)
setting = utils.from_json("json/setting.json")[args.dataset]
optimizer = torch.optim.Adam(model.parameters(),
lr=setting["learning_rate"],
weight_decay=setting["weight_decay"])
fitter = Fitter(model, data, optimizer)
history = fitter.run(verbose=args.verbose)
reward = max(history.val.acc)
return reward
def load_db(filename):
db_file = open(filename, "r")
content = db_file.read()
item_array = from_json(content)
item_dict = {}
for item in item_array:
item_dict[item["id"]] = item
db_file.close()
log("{0} was loaded:".format(filename))
log(item_dict)
return item_dict
def evaluate(params, dataset, device='cuda:0', val_test='test'):
data = Dataset(dataset)
gnn_graph = utils.build_gnn_graph(data, params)
model = GNN(gnn_graph).to(device)
# logger.info(dataset)
setting = utils.from_json("json/setting.json")[dataset]
optimizer = torch.optim.Adam(model.parameters(),
lr=setting["learning_rate"],
weight_decay=setting["weight_decay"])
fitter = Fitter(model, data[0].to(device), optimizer)
history = fitter.run(val_test=val_test, verbose=False)
return max(history.val.acc)
def post(self):
args = parser.parse_args()
id = 1 if len(flights) is 0 else int(max(flights.keys())) + 1
print(args)
flight = from_json(args["flight"])
print(flight)
print(id)
if (not "id" in flight.keys()) or (flight["id"] <= 0):
flight["id"] = id
flights[flight["id"]] = flight
self.save()
return flight #to_json(flight)
def from_dict(cls, d):
"""
Deserialize from dictionary.
"""
vallist = utils.from_json(d)
value = []
for v in vallist:
value.append(int(v))
com = CommitmentE(value)
return com
def deserialize(self, string, **kw):
"""
Function for serializing object => string.
This can be overwritten for custom
uses.
The default is to do nothing ('serializer'=None)
If the connection is intialized with 'serializer' set to
'json.gz', 'json', 'gz', or 'zip', we'll do the
transformations.
"""
serializer = kw.get('serializer', self._serializer)
if serializer == "json.gz":
return utils.from_json(utils.from_gz(string))
elif serializer == "json":
return utils.from_json(string)
elif serializer == "gz":
return utils.from_gz(string)
elif serializer == "zip":
return utils.from_zip(string)
elif serializer == "pickle":
return utils.from_pickle(obj)
elif serializer is not None:
raise NotImplementedError(
'Only json, gz, json.gz, zip, and pickle'
'are supported as serializers.')
return string
def post(self):
args = parser.parse_args()
id = 1 if len(users) is 0 else int(max(users.keys())) + 1
user = from_json(args["user"])
if (not "id" in user.keys()) or (user["id"] <= 0):
user["id"] = id
if not "isIbis" in user.keys():
user["isIbis"] = False
for u in users.values():
if u["username"] == user["username"]:
return u
users[user["id"]] = copy.deepcopy(user)
user["password"] = ""
self.save()
return user
def _cast_ballot(self, election_id, username, password, need_login=True, check_user_logged_in=False):
"""
check_user_logged_in looks for the "you're already logged" message
"""
# vote by preparing a ballot via the server-side encryption
response = self.app.post("/helios/elections/%s/encrypt-ballot" % election_id, {
'answers_json': utils.to_json([[1]])})
self.assertContains(response, "answers")
# parse it as an encrypted vote with randomness, and make sure randomness is there
the_ballot = utils.from_json(response.testbody)
assert the_ballot['answers'][0].has_key('randomness'), "no randomness"
assert len(the_ballot['answers'][0]['randomness']) == 2, "not enough randomness"
# parse it as an encrypted vote, and re-serialize it
ballot = datatypes.LDObject.fromDict(utils.from_json(response.testbody), type_hint='legacy/EncryptedVote')
encrypted_vote = ballot.serialize()
# cast the ballot
response = self.app.post("/helios/elections/%s/cast" % election_id, {
'encrypted_vote': encrypted_vote})
self.assertRedirects(response, "%s/helios/elections/%s/cast_confirm" % (settings.GET_SECURE_URL_HOST(request), election_id))
cast_confirm_page = response.follow()
if need_login:
if check_user_logged_in:
self.assertContains(cast_confirm_page, "You are logged in as")
self.assertContains(cast_confirm_page, "requires election-specific credentials")
# set the form
login_form = cast_confirm_page.form
login_form['voter_id'] = username
login_form['password'] = password
# we skip that intermediary page now
# cast_confirm_page = login_form.submit()
response = login_form.submit()
# self.assertRedirects(cast_confirm_page, "/helios/elections/%s/cast_confirm" % election_id)
# cast_confirm_page = cast_confirm_page.follow()
else:
# here we should be at the cast-confirm page and logged in
self.assertContains(cast_confirm_page, "VOTE con esta papeleta")
# confirm the vote, now with the actual form
cast_form = cast_confirm_page.form
if 'status_update' in cast_form.fields.keys():
cast_form['status_update'] = False
response = cast_form.submit()
self.assertRedirects(response, "%s/helios/elections/%s/cast_done" % (settings.URL_HOST, election_id))
# at this point an email should have gone out to the user
# at position num_messages after, since that was the len() before we cast this ballot
email_message = mail.outbox[len(mail.outbox) - 1]
url = re.search('http://[^/]+(/[^ \n]*)', email_message.body).group(1)
# check that we can get at that URL
if not need_login:
# confusing piece: if need_login is True, that means it was a public election
# that required login before casting a ballot.
# so if need_login is False, it was a private election, and we do need to re-login here
# we need to re-login if it's a private election, because all data, including ballots
# is otherwise private
login_page = self.app.get("/helios/elections/%s/password_voter_login" % election_id)
# if we redirected, that's because we can see the page, I think
if login_page.status_int != 302:
login_form = login_page.form
# try with extra spaces
login_form['voter_id'] = ' ' + username + ' '
login_form['password'] = '******' + password + ' '
login_form.submit()
response = self.app.get(url)
self.assertContains(response, ballot.hash)
self.assertContains(response, html_escape(encrypted_vote))
# if we request the redirect to cast_done, the voter should be logged out, but not the user
response = self.app.get("/helios/elections/%s/cast_done" % election_id)
def _setup_complete_election(self, election_params={}):
"do the setup part of a whole election"
# a bogus call to set up the session
self.client.get("/")
# REPLACE with params?
self.setup_login()
# create the election
full_election_params = {
"short_name" : "test-complete",
"name" : "Test Complete",
"description" : "A complete election test",
"election_type" : "referendum",
"use_voter_aliases": "0",
"use_advanced_audit_features": "1",
"private_p" : "False"}
# override with the given
full_election_params.update(election_params)
response = self.client.post("/helios/elections/new", full_election_params)
# we are redirected to the election, let's extract the ID out of the URL
election_id = re.search('/elections/([^/]+)/', str(response['Location'])).group(1)
# helios is automatically added as a trustee
# check that helios is indeed a trustee
response = self.client.get("/helios/elections/%s/trustees/view" % election_id)
self.assertContains(response, "Trustee #1")
# add a few voters with an improperly placed email address
FILE = "helios/fixtures/voter-badfile.csv"
voters_file = open(FILE)
response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'voters_file': voters_file})
voters_file.close()
self.assertContains(response, "HOLD ON")
# add a few voters, via file upload
# this file now includes a UTF-8 encoded unicode character
# yes I know that's not how you spell Ernesto.
# I just needed some unicode quickly.
FILE = "helios/fixtures/voter-file.csv"
voters_file = open(FILE)
response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'voters_file': voters_file})
voters_file.close()
self.assertContains(response, "first few rows of this file")
# now we confirm the upload
response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'confirm_p': "1"})
self.assertRedirects(response, "/helios/elections/%s/voters/list" % election_id)
# and we want to check that there are now voters
response = self.client.get("/helios/elections/%s/voters/" % election_id)
NUM_VOTERS = 4
self.assertEquals(len(utils.from_json(response.content)), NUM_VOTERS)
# let's get a single voter
single_voter = models.Election.objects.get(uuid = election_id).voter_set.all()[0]
response = self.client.get("/helios/elections/%s/voters/%s" % (election_id, single_voter.uuid))
self.assertContains(response, '"uuid": "%s"' % single_voter.uuid)
response = self.client.get("/helios/elections/%s/voters/foobar" % election_id)
self.assertEquals(response.status_code, 404)
# add questions
response = self.client.post("/helios/elections/%s/save_questions" % election_id, {
'questions_json': utils.to_json([{"answer_urls": [None,None], "answers": ["Alice", "Bob"], "choice_type": "approval", "max": 1, "min": 0, "question": "Who should be president?", "result_type": "absolute", "short_name": "Who should be president?", "tally_type": "homomorphic"}]),
'csrf_token': self.client.session['csrf_token']})
self.assertContains(response, "SUCCESS")
# freeze election
response = self.client.post("/helios/elections/%s/freeze" % election_id, {
"csrf_token" : self.client.session['csrf_token']})
self.assertRedirects(response, "/helios/elections/%s/view" % election_id)
# email the voters
num_messages_before = len(mail.outbox)
response = self.client.post("/helios/elections/%s/voters/email" % election_id, {
"csrf_token" : self.client.session['csrf_token'],
"subject" : "your password",
"body" : "time to vote",
"suppress_election_links" : "0",
"send_to" : "all"
})
self.assertRedirects(response, "/helios/elections/%s/view" % election_id)
num_messages_after = len(mail.outbox)
self.assertEquals(num_messages_after - num_messages_before, NUM_VOTERS)
email_message = mail.outbox[num_messages_before]
assert "your password" in email_message.subject, "bad subject in email"
# get the username and password
username = re.search('voter ID: (.*)', email_message.body).group(1)
password = re.search('password: (.*)', email_message.body).group(1)
# now log out as administrator
self.clear_login()
self.assertEquals(self.client.session.has_key('user'), False)
# return the voter username and password to vote
return election_id, username, password
def test_get_election_voters_raw(self):
response = self.client.get("/helios/elections/%s/voters/" % self.election.uuid, follow=False)
self.assertEquals(len(utils.from_json(response.content)), self.election.num_voters)
def nas(logger):
SPACE = space.ARCH_SPACE if args.arch_json is None else {}
if args.quantize is True:
SPACE = utils.join_space(SPACE, space.QUANT_SPACE)
if args.fpga is True:
SPACE = utils.join_space(SPACE, space.HW_SPACE)
fpga_info = utils.from_json(args.fpga_info_json)
fpga.load_fpga_info(fpga_info)
if len(SPACE) == 0:
print("Space is empty!")
exit()
pattern = [len(params) for params in SPACE.values()] * args.layers
device = 'cpu' if torch.cuda.is_available() is False else \
'cuda:{}'.format(args.gpu)
logger.info("using device {}".format(device))
controller = Controller(pattern, seed=args.seed)
best_reward, child_id = 0, 0
best_sample = 0
# reward_history = []
dataset = Dataset(args.dataset)
data = dataset[0].to(device)
# utils.display_dataset(dataset)
result = {f"Layer {i}": [] for i in range(args.layers)}
result["reward"] = []
result["time"] = []
registry = {}
gpu_calls = 0
if args.arch_json is not None:
arch_params = utils.from_json(args.arch_json)
def get_accuracy(params):
nonlocal gpu_calls
gpu_calls += 1
gnn_graph = utils.build_gnn_graph(dataset, params)
model = GNN(gnn_graph).to(device)
setting = utils.from_json("json/setting.json")[args.dataset]
optimizer = torch.optim.Adam(model.parameters(),
lr=setting["learning_rate"],
weight_decay=setting["weight_decay"])
fitter = Fitter(model, data, optimizer)
history = fitter.run(verbose=args.verbose)
reward = max(history.val.acc)
return reward
def fpga_analysis(params):
gnn_graph = utils.build_gnn_graph(dataset, params)
fpga_model = FPGAModel(data, gnn_graph)
return fpga_model.validate(Resource(args.rLUT, args.rReg, args.rDSP),
args.rCycle)
def get_reward(rollout, params):
if args.register is True:
key = tuple(rollout)
if key in registry:
return registry[key]
if args.fpga is True and fpga_analysis(params) is False:
reward = 0
else:
reward = get_accuracy(params)
return reward
episodes = args.episodes if args.gpu_eps is None else 2**32
for eps in range(episodes):
logger.info(f"Episode {eps+1}...")
if args.random is True:
rollout = controller.random_sample()
else:
rollout = controller.sample()
reward_batch = []
for r in rollout:
start_time = time.time()
child_id += 1
params = parse_rollout(r, SPACE)
if args.arch_json is not None:
params = utils.join_params(params, arch_params)
reward = get_reward(r, params)
[
result[f"Layer {i}"].append(params[i])
for i in range(len(params))
]
result["reward"].append(reward)
if reward > best_reward:
best_reward = reward
best_sample = child_id
reward_batch.append(reward)
elasped_time = time.time() - start_time
result["time"].append(elasped_time)
logger.info(f"Child Network: {child_id} (best: {best_sample}), "
# f"Rollout: {params}, "
f"Reward: {reward:.4f} ({best_reward:.4f}), "
f"Time: {elasped_time:.2f} s, "
f"GPU Calls: {gpu_calls}")
registry[tuple(r)] = reward
if args.gpu_eps is not None and gpu_calls >= args.gpu_eps:
# print(args.gpu_eps, gpu_calls)
break
if args.random is False:
controller.update(reward_batch)
df = pd.DataFrame(result)
return df
def one_election_save_questions(request, election): election.questions = utils.from_json(request.POST['questions_json']); election.save() # always a machine API return SUCCESS
def election_new_3(request):
"""
Create the new election.
trustees is a JSON list
"""
name = request.POST['name']
if request.POST.has_key('trustee_list'):
trustee = request.POST['trustee_list'].split(",")
else:
trustee = request.POST.getlist('trustee')
public_key = request.POST.get('public_key', None)
private_key = request.POST.get('private_key', None)
voting_starts_at = request.POST.get('voting_starts_at', None)
voting_ends_at = request.POST.get('voting_ends_at', None)
# election type is homomorphic. The type of the election determines
# how votes are tallied and verified.
ballot_type = request.POST.get('ballot_type', 'homomorphic')
tally_type = request.POST.get('tally_type', 'homomorphic')
# we need a list of admins, or at least a public key
if len(trustee) == 0 and not public_key:
return HttpResponseServerError(
'Need a list of trustees or a public key')
# create an election
if public_key and public_key != "":
pk = algs.EGPublicKey.from_dict(utils.from_json(public_key))
else:
pk = None
if private_key and private_key != "":
sk = algs.EGSecretKey.from_dict(utils.from_json(private_key))
else:
sk = None
election = Election.objects.create(
ballot_type=ballot_type,
tally_type=tally_type,
name=name,
admin=get_user(request),
api_client=get_api_client(request),
# voting_starts_at = utils.string_to_datetime(voting_starts_at),
# voting_ends_at = utils.string_to_datetime(voting_ends_at),
public_key=pk,
private_key=sk)
## FIXME: transaction!
# go through the trustees
if len(trustee) > 0:
for t in trustee:
if t.strip() == "":
continue
# create the keyshare
keyshare = KeyShare.objects.create(election=election, email=t)
keyshare.generate_password()
keyshare.save()
# send out the email
## NO LONGER BY DEFAULT - must send the mail manually
# send_trustees_email(election, 'Trustee for Election %s' % election.name, 'You have been designated as a trustee of the Helios Election "%s".' % election.name)
# user or api_client?
if get_user(request):
return HttpResponseRedirect(
reverse(one_election_view, args=[election.election_id]))
else:
return HttpResponse(str(election.election_id))
def election_get(self, election_id):
return electionalgs.Election.fromJSONDict(utils.from_json(self.get("/elections/%s/" % election_id)))
def test_get_election_voters_raw(self):
response = self.client.get("/helios/elections/%s/voters/" % self.election.uuid, follow=False)
self.assertEquals(len(utils.from_json(response.content)), self.election.num_voters)
def load_fpga_info(info):
global fpga
for op, table in info.items():
fpga.load_operator(op, table)
return
def get_operator_info(op_type: str):
return fpga.get_operator_info(op_type)
def get_fpga_capacity():
return fpga.capacity
def get_cycle():
return fpga.cycle
def show_base():
for k, v in fpga.ops.items():
print(k)
if __name__ == "__main__":
import utils
fpga_info = utils.from_json("./json/fpga/fpga_info.json")
load_fpga_info(fpga_info)
print(fpga.ops)
from utils import from_json
from glob import glob
gene_to_length = from_json("hg19_with_GFP/gene_to_length.json")
"""
Read HTSeq-count output files and create a corresponding _fpkm.txt file for each
"""
for fname in glob("output_with_GFP/*.txt"):
gene_to_count = {}
total_count = 0
with open(fname, 'r') as ropen:
for line in ropen:
sl = line.strip().split()
if len(sl) == 0: continue
gene, count = sl
if gene not in gene_to_length: continue
count = int(count)
gene_to_count[gene] = count
total_count += count
output_fname = fname.replace(".txt", "_fpkm.txt")
with open(output_fname, 'w+') as wopen:
for gene, count in gene_to_count.items():
length = gene_to_length[gene]
fpkm = count / (length / 1e3) / (total_count / 1e6)
wopen.write(f"{gene} {fpkm}\n")
def nas(logger):
SPACE = space.ARCH_SPACE
if len(SPACE) == 0:
print("Space is empty!")
exit()
pattern = [len(params) for params in SPACE.values()] * args.layers
device = 'cpu' if torch.cuda.is_available() is False else \
'cuda:{}'.format(args.gpu)
logger.info("using device {}".format(device))
controller = Controller(pattern)
# reward_history = []
dataset = Dataset(args.dataset)
data = dataset[0].to(device)
# utils.display_dataset(dataset)
result = {f"Layer {i}": [] for i in range(args.layers)}
result["reward"] = []
result["time"] = []
# searching architectures
def get_accuracy(params):
gnn_graph = utils.build_gnn_graph(dataset, params)
model = GNN(gnn_graph).to(device)
setting = utils.from_json("json/setting.json")[args.dataset]
optimizer = torch.optim.Adam(model.parameters(),
lr=setting["learning_rate"],
weight_decay=setting["weight_decay"])
fitter = Fitter(model, data, optimizer)
history = fitter.run(verbose=args.verbose)
reward = max(history.val.acc)
return reward
arch_id, best_arch, best_accuracy = 0, 0, 0
registry = {}
for eps in range(args.arch_episodes):
logger.info(f"Architecture Episode {eps+1}...")
if args.random is True:
rollout = controller.random_sample()
else:
rollout = controller.sample()
reward_batch = []
for r in rollout:
start_time = time.time()
arch_id += 1
arch_params = parse_rollout(r, SPACE)
key = tuple(r)
reward = registry[key] if key in registry else \
get_accuracy(arch_params)
[
result[f"Layer {i}"].append(arch_params[i])
for i in range(len(arch_params))
]
result["reward"].append(reward)
if reward > best_accuracy:
best_accuracy = reward
best_arch = arch_id
best_arch_params = arch_params
reward_batch.append(reward)
elasped_time = time.time() - start_time
result["time"].append(elasped_time)
logger.info(
f"Child Architecture: {arch_id} ", f"(best: {best_arch}), "
f"Rollout: {arch_params}, "
f"Reward: {reward:.4f} ({best_accuracy:.4f}), "
f"Time: {elasped_time:.2f} s")
registry[tuple(r)] = reward
if args.random is False:
controller.update(reward_batch)
# searching hardware
SPACE = space.HW_SPACE
if args.quantize is True:
SPACE = utils.join_space(SPACE, space.QUANT_SPACE)
fpga_info = utils.from_json(args.fpga_info_json)
fpga.load_fpga_info(fpga_info)
if len(SPACE) == 0:
print("Space is empty!")
exit()
pattern = [len(params) for params in SPACE.values()] * args.layers
controller = Controller(pattern)
def get_hw_reward(params):
if fpga_analysis(params) is False:
return 0
if args.quantize is True:
return get_accuracy(params)
return best_accuracy
def fpga_analysis(params):
gnn_graph = utils.build_gnn_graph(dataset, params)
fpga_model = FPGAModel(data, gnn_graph)
return fpga_model.validate(Resource(args.rLUT, args.rReg, args.rDSP),
args.rCycle)
hw_id, best_hw, best_hw_reward = 0, 0, 0
registry = {}
for eps in range(args.hardware_episodes):
logger.info(f"Hardware Episode {eps+1}...")
if args.random is True:
rollout = controller.random_sample()
else:
rollout = controller.sample()
reward_batch = []
for r in rollout:
start_time = time.time()
hw_id += 1
hw_params = parse_rollout(r, SPACE)
params = join_params(best_arch_params, hw_params)
key = tuple(r)
reward = registry[key] if key in registry else \
get_hw_reward(params)
[
result[f"Layer {i}"].append(params[i])
for i in range(len(params))
]
result["reward"].append(reward)
if reward > best_hw_reward:
best_hw_reward = reward
best_hw = hw_id
reward_batch.append(reward)
elasped_time = time.time() - start_time
result["time"].append(elasped_time)
logger.info(
f"Child Hardware: {hw_id} ", f"(best: {best_hw}), "
f"Rollout: {hw_params}, "
f"Reward: {reward:.4f} ({best_hw_reward:.4f}), "
f"Time: {elasped_time:.2f} s")
registry[key] = reward
if args.random is False:
controller.update(reward_batch)
df = pd.DataFrame(result)
return df
def _cast_ballot(self, election_id, username, password, need_login=True, check_user_logged_in=False):
"""
check_user_logged_in looks for the "you're already logged" message
"""
# vote by preparing a ballot via the server-side encryption
response = self.app.post("/helios/elections/%s/encrypt-ballot" % election_id, {
'answers_json': utils.to_json([[1]])})
self.assertContains(response, "answers")
# parse it as an encrypted vote, and re-serialize it
ballot = datatypes.LDObject.fromDict(utils.from_json(response.testbody), type_hint='legacy/EncryptedVote')
encrypted_vote = ballot.serialize()
# cast the ballot
response = self.app.post("/helios/elections/%s/cast" % election_id, {
'encrypted_vote': encrypted_vote})
self.assertRedirects(response, "%s/helios/elections/%s/cast_confirm" % (settings.SECURE_URL_HOST, election_id))
cast_confirm_page = response.follow()
if need_login:
if check_user_logged_in:
self.assertContains(cast_confirm_page, "You are logged in as")
self.assertContains(cast_confirm_page, "requires election-specific credentials")
# set the form
login_form = cast_confirm_page.form
login_form['voter_id'] = username
login_form['password'] = password
cast_confirm_page = login_form.submit()
self.assertRedirects(cast_confirm_page, "/helios/elections/%s/cast_confirm" % election_id)
cast_confirm_page = cast_confirm_page.follow()
# here we should be at the cast-confirm page and logged in
self.assertContains(cast_confirm_page, "I am ")
# confirm the vote, now with the actual form
cast_form = cast_confirm_page.form
if 'status_update' in cast_form.fields.keys():
cast_form['status_update'] = False
response = cast_form.submit()
self.assertRedirects(response, "%s/helios/elections/%s/cast_done" % (settings.URL_HOST, election_id))
# at this point an email should have gone out to the user
# at position num_messages after, since that was the len() before we cast this ballot
email_message = mail.outbox[len(mail.outbox) - 1]
url = re.search('http://[^/]+(/[^ \n]*)', email_message.body).group(1)
# check that we can get at that URL
if not need_login:
# confusing piece: if need_login is True, that means it was a public election
# that required login before casting a ballot.
# so if need_login is False, it was a private election, and we do need to re-login here
# we need to re-login if it's a private election, because all data, including ballots
# is otherwise private
login_page = self.app.get("/helios/elections/%s/password_voter_login" % election_id)
# if we redirected, that's because we can see the page, I think
if login_page.status_int != 302:
login_form = login_page.form
# try with extra spaces
login_form['voter_id'] = ' ' + username + ' '
login_form['password'] = '******' + password + ' '
login_form.submit()
response = self.app.get(url)
self.assertContains(response, ballot.hash)
self.assertContains(response, html_escape(encrypted_vote))
# if we request the redirect to cast_done, the voter should be logged out, but not the user
response = self.app.get("/helios/elections/%s/cast_done" % election_id)
def params(self):
params_json = self.get("/elections/params")
return algs.ElGamal.fromJSONDict(utils.from_json(params_json))
def _setup_complete_election(self, election_params={}):
"do the setup part of a whole election"
# a bogus call to set up the session
self.client.get("/")
# REPLACE with params?
self.setup_login()
# create the election
full_election_params = {
"short_name" : "test-complete",
"name" : "Test Complete",
"description" : "A complete election test",
"election_type" : "referendum",
"use_voter_aliases": "0",
"use_advanced_audit_features": "1",
"private_p" : "0"}
# override with the given
full_election_params.update(election_params)
response = self.client.post("/helios/elections/new", full_election_params)
# we are redirected to the election, let's extract the ID out of the URL
election_id = re.search('/elections/([^/]+)/', str(response['Location'])).group(1)
# helios is automatically added as a trustee
# check that helios is indeed a trustee
response = self.client.get("/helios/elections/%s/trustees/view" % election_id)
self.assertContains(response, "Trustee #1")
# add a few voters, via file upload
FILE = "helios/fixtures/voter-file.csv"
voters_file = open(FILE)
response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'voters_file': voters_file})
voters_file.close()
self.assertContains(response, "first few rows of this file")
# now we confirm the upload
response = self.client.post("/helios/elections/%s/voters/upload" % election_id, {'confirm_p': "1"})
self.assertRedirects(response, "/helios/elections/%s/voters/list" % election_id)
# and we want to check that there are now voters
response = self.client.get("/helios/elections/%s/voters/" % election_id)
NUM_VOTERS = 4
self.assertEquals(len(utils.from_json(response.content)), NUM_VOTERS)
# let's get a single voter
single_voter = models.Election.objects.get(uuid = election_id).voter_set.all()[0]
response = self.client.get("/helios/elections/%s/voters/%s" % (election_id, single_voter.uuid))
self.assertContains(response, '"uuid": "%s"' % single_voter.uuid)
response = self.client.get("/helios/elections/%s/voters/foobar" % election_id)
self.assertEquals(response.status_code, 404)
# add questions
response = self.client.post("/helios/elections/%s/save_questions" % election_id, {
'questions_json': utils.to_json([{"answer_urls": [None,None], "answers": ["Alice", "Bob"], "choice_type": "approval", "max": 1, "min": 0, "question": "Who should be president?", "result_type": "absolute", "short_name": "Who should be president?", "tally_type": "homomorphic"}]),
'csrf_token': self.client.session['csrf_token']})
self.assertContains(response, "SUCCESS")
# freeze election
response = self.client.post("/helios/elections/%s/freeze" % election_id, {
"csrf_token" : self.client.session['csrf_token']})
self.assertRedirects(response, "/helios/elections/%s/view" % election_id)
# email the voters
num_messages_before = len(mail.outbox)
response = self.client.post("/helios/elections/%s/voters/email" % election_id, {
"csrf_token" : self.client.session['csrf_token'],
"subject" : "your password",
"body" : "time to vote",
"suppress_election_links" : "0",
"send_to" : "all"
})
self.assertRedirects(response, "/helios/elections/%s/view" % election_id)
num_messages_after = len(mail.outbox)
self.assertEquals(num_messages_after - num_messages_before, NUM_VOTERS)
email_message = mail.outbox[num_messages_before]
assert "your password" in email_message.subject, "bad subject in email"
# get the username and password
username = re.search('voter ID: (.*)', email_message.body).group(1)
password = re.search('password: (.*)', email_message.body).group(1)
# now log out as administrator
self.clear_login()
self.assertEquals(self.client.session.has_key('user'), False)
# return the voter username and password to vote
return election_id, username, password
# required = getattr(rResource, r)
# usage = getattr(self.usage, r)
# score = (required - usage) / required if usage > required else 0
# if usage == 0:
# score = 0
# else:
# score = math.log(required / usage)
# score = 1 - (usage / required)
# metric += 0.25 * score
# score = (rCycle - self.cycle) / rCycle if self.cycle > rCycle \
# else 0
# score = math.log(rCycle / self.cycle)
# score = 1 - (self.cycle / rCycle)
# metric += 0.25 * score
return (self.usage < rResource) and (self.cycle < rCycle)
# return metric
if __name__ == "__main__":
import utils
from dataset import cora
from .fpga import load_fpga_info
fpga_info = utils.from_json("./json/fpga/fpga_info.json")
load_fpga_info(fpga_info)
gnn_graph = utils.from_json("./json/sample.json")
fpga_model = FPGAModel(cora[0], gnn_graph)
print(fpga_model.usage)
print(fpga_model.cycle)
print(fpga_model.validate())
def election_new_3(request):
"""
Create the new election.
trustees is a JSON list
"""
name = request.POST['name']
if request.POST.has_key('trustee_list'):
trustee = request.POST['trustee_list'].split(",")
else:
trustee = request.POST.getlist('trustee')
public_key = request.POST.get('public_key', None)
private_key = request.POST.get('private_key', None)
voting_starts_at = request.POST.get('voting_starts_at', None)
voting_ends_at = request.POST.get('voting_ends_at', None)
# election type is homomorphic. The type of the election determines
# how votes are tallied and verified.
ballot_type = request.POST.get('ballot_type', 'homomorphic')
tally_type = request.POST.get('tally_type', 'homomorphic')
# we need a list of admins, or at least a public key
if len(trustee) == 0 and not public_key:
return HttpResponseServerError('Need a list of trustees or a public key')
# create an election
if public_key and public_key != "":
pk = algs.EGPublicKey.from_dict(utils.from_json(public_key))
else:
pk = None
if private_key and private_key != "":
sk = algs.EGSecretKey.from_dict(utils.from_json(private_key))
else:
sk = None
election = Election.objects.create(ballot_type = ballot_type, tally_type = tally_type, name = name,
admin = get_user(request), api_client= get_api_client(request),
# voting_starts_at = utils.string_to_datetime(voting_starts_at),
# voting_ends_at = utils.string_to_datetime(voting_ends_at),
public_key = pk, private_key = sk)
## FIXME: transaction!
# go through the trustees
if len(trustee) > 0:
for t in trustee:
if t.strip() == "":
continue
# create the keyshare
keyshare = KeyShare.objects.create(election = election, email = t)
keyshare.generate_password()
keyshare.save()
# send out the email
## NO LONGER BY DEFAULT - must send the mail manually
# send_trustees_email(election, 'Trustee for Election %s' % election.name, 'You have been designated as a trustee of the Helios Election "%s".' % election.name)
# user or api_client?
if get_user(request):
return HttpResponseRedirect(reverse(one_election_view, args=[election.election_id]))
else:
return HttpResponse(str(election.election_id))
def one_election_save_questions(request, election):
election.questions = utils.from_json(request.POST['questions_json'])
election.save()
# always a machine API
return SUCCESS
