def worker(queue, eviction_policy, processes, threads):
timings = {'get': [], 'set': [], 'delete': []}
cache = Cache('tmp', eviction_policy=eviction_policy)
for index, (action, key, value) in enumerate(iter(queue.get, None)):
start = time.time()
if action == 'set':
cache.set(key, value, expire=EXPIRE)
elif action == 'get':
result = cache.get(key)
else:
assert action == 'delete'
cache.delete(key)
stop = time.time()
if action == 'get' and processes == 1 and threads == 1 and EXPIRE is None:
assert result == value
if index > WARMUP:
timings[action].append(stop - start)
queue.put(timings)
cache.close()
class FileCache(BaseCache):
""" BaseCache implementation using files to store the data.
This implementation uses diskcache.Cache
see http://www.grantjenks.com/docs/diskcache/api.html#cache for more
informations
This cache requires you to install diskcache using `pip install diskcache`
"""
def __init__(self, path, **settings):
""" Constructor
Arguments:
path {String} -- The path on the disk to save the data
settings {dict} -- The settings values for diskcache
"""
from diskcache import Cache
self._cache = Cache(path, **settings)
def __del__(self):
""" Close the connection as the cache instance is deleted.
Safe to use as there are no circular ref.
"""
self._cache.close()
def set(self, key, value, timeout=300):
expire_time = None if timeout == 0 else timeout
self._cache.set(_hash(key), value, expire=expire_time)
def get(self, key, default=None):
return self._cache.get(_hash(key), default)
def invalidate(self, key):
self._cache.delete(_hash(key))
def worker(queue, eviction_policy, processes, threads):
timings = co.defaultdict(list)
cache = Cache('tmp', eviction_policy=eviction_policy)
for index, (action, key, value) in enumerate(iter(queue.get, None)):
start = time.time()
try:
if action == 'set':
cache.set(key, value, expire=EXPIRE)
elif action == 'get':
result = cache.get(key)
else:
assert action == 'delete'
cache.delete(key)
except Timeout:
miss = True
else:
miss = False
stop = time.time()
if (action == 'get' and processes == 1 and threads == 1
and EXPIRE is None):
assert result == value
if index > WARMUP:
delta = stop - start
timings[action].append(delta)
if miss:
timings[action + '-miss'].append(delta)
queue.put(timings)
cache.close()
def worker(queue, eviction_policy):
timings = {'get': [], 'set': [], 'delete': []}
cache = Cache('tmp', eviction_policy=eviction_policy)
for index, (action, key, value) in enumerate(iter(queue.get, None)):
start = time.time()
if action == 'set':
cache.set(key, value, expire=EXPIRE)
elif action == 'get':
result = cache.get(key)
else:
assert action == 'delete'
cache.delete(key)
stop = time.time()
if action == 'get' and PROCESSES == 1 and THREADS == 1 and EXPIRE is None:
assert result == value
if index > WARMUP:
timings[action].append(stop - start)
queue.put(timings)
cache.close()
class CacheProxy:
def __init__(self, script):
self.config = get_configs()
collectors_dir = self.config.get('base', 'collectors_dir')
self.cache = Cache(
os.path.join(collectors_dir, 'cache/script/', script))
def get(self, key):
return self.cache.get(key)
def set(self, key, value):
self.cache.set(key, value)
def delete(self, key):
self.cache.delete(key)
def close(self):
self.cache.close()
def counter_to_gauge(self, key, value):
last_value = self.get(key)
self.set(key, value)
if last_value is None:
return None
gauge = value - last_value
if gauge < 0 or gauge > last_value:
return None
return gauge
class DiscCacheWrapper():
DISC_CACHE_PATH = "./tmp/flickr-auth-disc-cache"
def __init__(self):
self.disc_cache = Cache(directory=DiscCacheWrapper.DISC_CACHE_PATH)
def get(self, key):
# This can raise a diskcache.Timeout error if it fails to talk to its database
return self.disc_cache.get(key=key, default=None, retry=False)
def set(self, key, value, timeout=None):
# This can raise a diskcache.Timeout error if it fails to talk to its database
self.disc_cache.set(key=key, value=value, expire=timeout, retry=False)
def delete(self, key):
self.disc_cache.delete(key=key)
class Cache(object):
def __init__(self):
try:
self.cache = DC('./tmp')
except Exception as ex:
print('Get an exception with diskcache open: {}'.format(ex))
self.cache = None
def __del__(self):
try:
self.cache.close()
except Exception as ex:
print('Get an exception with diskcache close: {}'.format(ex))
def set(self, key, value):
if self.cache is not None:
self.cache.set(key, BytesIO(value), read=True, tag=u'data')
def get(self, key):
if self.cache is not None:
value = self.cache.get(key, default=b'', read=True, tag=True)
if value is not None and value != b'':
return value
return None
def pop(self, key):
if self.cache is not None:
value = self.cache.pop(key, default=b'', read=True, tag=True)
if value is not None and value != b'':
return value
return None
def delete(self, key):
if self.cache is not None:
self.cache.delete(key)
def create_index(self):
if self.cache is not None:
self.cache.create_tag_index()
return self.cache.tag_index
return None
def clear_all(self):
if self.cache is not None:
self.cache.clear()
class RouteHandler(object):
def __init__(self, loop, messenger, database):
self._loop = loop
self._messenger = messenger
self._database = database
self.cache = Cache()
async def create_election(self, request):
private_key, public_key, user = await self._authorize(request)
body = await decode_request(request)
required_fields = [
'name', 'description', 'start_timestamp', 'end_timestamp',
'results_permission', 'can_change_vote', 'can_show_realtime',
'voting_options', 'poll_book'
]
validate_fields(required_fields, body)
if user.get('type') != 'ADMIN' and user.get('type') != 'SUPERADMIN':
raise ApiForbidden('Voter must be an admin or superadmin')
election_id = uuid.uuid1().hex
voting_options = body.get('voting_options')
admin = await self._database.fetch_voter_resource(public_key=public_key
)
for voting_option in voting_options:
if voting_option.get('name').upper(
) == "NULL" or voting_option.get('name').upper() == "BLANK":
raise ApiInternalError('NULL and BLANK are default options')
voting_options.append({"name": "NULL", "description": "VOTE NULL"})
voting_options.append({"name": "BLANK", "description": "VOTE BLANK"})
await self._messenger.send_create_election_transaction(
private_key=private_key,
election_id=election_id,
name=body.get('name'),
description=body.get('description'),
start_timestamp=body.get('start_timestamp'),
end_timestamp=body.get('end_timestamp'),
results_permission=body.get('results_permission'),
can_change_vote=body.get('can_change_vote'),
can_show_realtime=body.get('can_show_realtime'),
admin_id=admin.get('voter_id'),
status=1,
timestamp=get_time())
for voting_option in voting_options:
voting_option_id = uuid.uuid1().hex
await self._messenger.send_create_voting_option_transaction(
private_key=private_key,
voting_option_id=voting_option_id,
name=voting_option.get('name'),
description=voting_option.get('description'),
election_id=election_id,
status=1,
timestamp=get_time())
await self._database.insert_voting_option_num_vote_resource(
voting_option_id=voting_option_id,
name=voting_option.get('name'),
election_id=election_id)
for poll_book in body.get('poll_book'):
await self._messenger.send_create_poll_registration_transaction(
private_key=private_key,
voter_id=poll_book.get('id'),
name=poll_book.get('name'),
election_id=election_id,
status=1,
timestamp=get_time())
return json_response({'data': 'Create election transaction submitted'})
async def create_voter(self, request):
body = await decode_request(request)
required_fields = ['voter_id', 'name', 'password']
validate_fields(required_fields, body)
if await self._database.is_voter_created(body.get("voter_id")
) is not None:
raise ApiConflict("Voter ID must be unique")
public_key, private_key = self._messenger.get_new_key_pair()
await self._messenger.send_create_voter_transaction(
private_key=private_key,
voter_id=body.get('voter_id'),
public_key=public_key,
name=body.get('name'),
created_at=get_time(),
type='VOTER')
encrypted_private_key = encrypt_private_key(request.app['aes_key'],
public_key, private_key)
hashed_password = hash_password(body.get('password'))
await self._database.create_auth_entry(public_key,
encrypted_private_key,
hashed_password)
user = {
'name': body.get('name'),
'voter_id': body.get('voter_id'),
'type': 'VOTER'
}
token = self.generate_auth_token(request.app['secret_key'], public_key,
user)
return json_response({'accessToken': token, 'user': user})
async def update_voter_type(self, request):
private_key, public_key, user = await self._authorize(request)
body = await decode_request(request)
required_fields = ['type']
validate_fields(required_fields, body)
voter_id = request.match_info.get('voterId', '')
if voter_id == '':
raise ApiBadRequest(
'The voter ID is a required query string parameter')
if user.get('type') != 'SUPERADMIN':
raise ApiForbidden('Forbidden')
voter = await self._database.fetch_voter_resource(voter_id=voter_id)
if voter is None:
raise ApiNotFound('No voter found')
if body.get('type') == 'ADMIN' and (voter.get('type') == 'ADMIN' or
voter.get('type') == 'SUPERADMIN'):
raise ApiConflict(
'Voter {} is already an admin or superadmin'.format(voter_id))
elif body.get('type') == 'VOTER' and voter.get('type') == 'VOTER':
raise ApiConflict('Voter {} is already a voter. '.format(voter_id))
auth_info = await self._database.fetch_auth_resource(
public_key=voter.get('public_key'))
voter_private_key = decrypt_private_key(
request.app['aes_key'], voter.get('public_key'),
auth_info.get('encrypted_private_key'))
await self._messenger.send_update_voter_transaction(
private_key=voter_private_key,
voter_id=voter_id,
public_key=voter.get('public_key'),
name=voter.get('name'),
created_at=get_time(),
type=body.get('type'))
return json_response({
'voter': {
'voter_id': voter_id,
'name': voter.get('name'),
'type': 'ADMIN'
}
})
async def create_vote(self, request):
body = await decode_request(request)
required_fields = []
validate_fields(required_fields, body)
private_key, public_key, user = await self._authorize(request)
voting_option_id = request.match_info.get('votingOptionId', '')
if voting_option_id == '':
raise ApiBadRequest(
'The voting option ID is a required query string parameter')
voter = await self._database.fetch_voter_resource(public_key=public_key
)
if voter is None:
raise ApiNotFound('Voter with the public_key '
'{} was not found'.format(public_key))
voting_option = await self._database.fetch_voting_option_resource(
voting_option_id=voting_option_id)
vo_count_vote = await self._database.fetch_voting_option_num_vote_resource(
voting_option_id=voting_option_id)
if voting_option is None:
raise ApiNotFound('Voting Option with the voting option id '
'{} was not found'.format(voting_option_id))
election_id = voting_option.get('election_id')
election = await self._database.fetch_election_resource(
election_id=election_id)
if election.get('status') == 0:
raise ApiBadRequest('Election with the election id '
'{} is cancelled'.format(election_id))
current_time = get_time()
if election.get('end_timestamp') < current_time or election.get(
'start_timestamp') > current_time:
raise ApiBadRequest('Not in election time.'.format())
poll_registration = await self._database.fetch_poll_book_registration(
voter_id=user.get('voter_id'), election_id=election_id)
if poll_registration is None:
raise ApiBadRequest(
'Voter is not registered in the poll book of the election with the id '
'{} .'.format(election_id))
num_votes_update = vo_count_vote.get('num_votes') + 1
await self._messenger.send_create_vote_transaction(
private_key=private_key,
vote_id=uuid.uuid1().hex,
timestamp=get_time(),
voter_id=voter.get('voter_id'),
election_id=voting_option.get('election_id'),
voting_option_id=voting_option_id)
await self._database.update_voting_option_num_vote_resource(
voting_option_id=voting_option_id, num_votes=num_votes_update)
return json_response({'data': 'Create vote transaction submitted'})
async def update_vote(self, request):
private_key, public_key, user = await self._authorize(request)
body = await decode_request(request)
required_fields = ['voting_option_id']
validate_fields(required_fields, body)
vote_id = request.match_info.get('voteId', '')
if vote_id == '':
raise ApiBadRequest(
'The vote ID is a required query string parameter')
vote = await self._database.fetch_vote_resource(vote_id=vote_id)
election_id = vote.get('election_id')
if vote is None:
raise ApiNotFound('Vote with the vote id '
'{} was not found'.format(vote_id))
if vote.get('voting_option_id') == body.get('voting_option_id'):
raise ApiBadRequest('Vote must be different.')
election = await self._database.fetch_election_resource(
election_id=election_id)
if election is None:
raise ApiNotFound('Election with the election id '
'{} was not found'.format(election_id))
if election.get('can_change_vote') == 0:
raise ApiInternalError(
'Election with the election id '
'{} was not found don\'t permit to change vote'.format(
election_id))
if election.get('can_change_vote') == 0:
raise ApiInternalError(
'Election with the election id '
'{} was not found don\'t permit to change vote'.format(
election_id))
current_time = get_time()
if election.get('end_timestamp') < current_time or election.get(
'start_timestamp') > current_time:
raise ApiBadRequest('Not in election time.'.format())
if election.get('admin_id') == user.get('voter_id'):
raise ApiBadRequest(
'User is not the owner of the election with the id '
'{} .'.format(election_id))
new_voting_option_id = body.get('voting_option_id')
old_voting_option_id = vote.get('voting_option_id')
old_num_vote = await self._database.fetch_voting_option_num_vote_resource(
voting_option_id=old_voting_option_id)
new_num_vote = await self._database.fetch_voting_option_num_vote_resource(
voting_option_id=new_voting_option_id)
num_votes_remove = old_num_vote.get('num_votes') - 1
num_votes_update = new_num_vote.get('num_votes') + 1
await self._messenger.send_update_vote_transaction(
private_key=private_key,
vote_id=vote_id,
timestamp=get_time(),
voting_option_id=new_voting_option_id)
# remove -1 to old voting option
await self._database.update_voting_option_num_vote_resource(
voting_option_id=old_voting_option_id, num_votes=num_votes_remove)
# add +1 to new voting option
await self._database.update_voting_option_num_vote_resource(
voting_option_id=new_voting_option_id, num_votes=num_votes_update)
return json_response({'data': 'Update Vote transaction submitted'})
async def update_election(self, request):
private_key, public_key, user = await self._authorize(request)
body = await decode_request(request)
election_id = request.match_info.get('electionId', '')
if election_id == '':
raise ApiBadRequest(
'The election ID is a required query string parameter')
election = await self._database.fetch_election_resource(
election_id=election_id)
if election is None:
raise ApiNotFound('Election with the election id '
'{} was not found'.format(election_id))
current_time = get_time()
if election.get('start_timestamp') < current_time:
raise ApiBadRequest('Election with the election id '
'{} already start.'.format(election_id))
if election.get('admin_id') != user.get('voter_id'):
raise ApiBadRequest(
'User is not the owner of the election with the id '
'{} .'.format(election_id))
await self._messenger.send_update_election_transaction(
private_key=private_key,
election_id=election_id,
name=body.get('name')
if body.get('name') is not None else election.get('name'),
description=body.get('description') if body.get('description')
is not None else election.get('description'),
start_timestamp=body.get('start_timestamp')
if body.get('start_timestamp') is not None else
election.get('start_timestamp'),
end_timestamp=body.get('end_timestamp')
if body.get('end_timestamp') is not None else
election.get('end_timestamp'),
results_permission=body.get('results_permission')
if body.get('results_permission') is not None else
election.get('results_permission'),
can_change_vote=body.get('can_change_vote')
if body.get('can_change_vote') is not None else
election.get('can_change_vote'),
can_show_realtime=body.get('can_show_realtime')
if body.get('can_show_realtime') is not None else
election.get('can_show_realtime'),
admin_id=user.get('voter_id'),
status=body.get('status')
if body.get('status') is not None else election.get('status'),
timestamp=get_time())
if body.get('voting_options') is not None:
for voting_option in body.get('voting_options'):
voting_option_id = uuid.uuid1().hex
await self._messenger.send_create_voting_option_transaction(
private_key=private_key,
voting_option_id=voting_option_id,
name=voting_option.get('name'),
description=voting_option.get('description'),
election_id=election_id,
status=1,
timestamp=get_time())
await self._database.insert_voting_option_num_vote_resource(
voting_option_id=voting_option_id,
name=voting_option.get('name'),
election_id=election_id)
if body.get('poll_book') is not None:
for poll_book in body.get('poll_book'):
await self._messenger.send_create_poll_registration_transaction(
private_key=private_key,
voter_id=poll_book.get('id'),
name=poll_book.get('name'),
election_id=election_id,
status=1,
timestamp=get_time())
return json_response({'data': 'Update Election transaction submitted'})
async def get_election(self, request):
private_key, public_key, user = await self._authorize(request)
election_id = request.match_info.get('electionId', '')
if election_id == '':
raise ApiBadRequest(
'The election ID is a required query string parameter')
if 'asAdmin' in request.rel_url.query:
election = await self._database.fetch_election_with_can_vote_resource_admin(
voter_id=user.get('voter_id'), election_id=election_id)
else:
election = await self._database.fetch_election_with_can_vote_resource(
voter_id=user.get('voter_id'), election_id=election_id)
if election is None:
raise ApiNotFound('Election with the election id '
'{} was not found'.format(election_id))
if election.get('results_permission') != 'PUBLIC':
if election.get('can_vote') is False and election.get(
'admin_id') != user.get('voter_id'):
raise ApiForbidden(
'Voter is not registered in the poll book of the election with the id '
'{}.'.format(election_id))
return json_response(election)
async def get_election_votes(self, request):
private_key, public_key, user = await self._authorize(request)
election_id = request.match_info.get('electionId', '')
if election_id == '':
raise ApiBadRequest(
'The election ID is a required query string parameter')
number_of_votes = await self._database.fetch_number_of_votes(
election_id=election_id)
if number_of_votes is None:
raise ApiNotFound('No voting options with the election id '
'{} was not found'.format(election_id))
election = await self._database.fetch_election_resource(
election_id=election_id)
if election is None:
raise ApiNotFound('Election with the election id '
'{} was not found'.format(election_id))
if election.get('results_permission') != 'PUBLIC':
poll_registration = await self._database.fetch_poll_book_registration(
voter_id=user.get('voter_id'), election_id=election_id)
if poll_registration is None and election.get(
'admin_id') != user.get('voter_id'):
raise ApiBadRequest(
'Voter is not registered in the poll book of the election with the id '
'{} .'.format(election_id))
return json_response(number_of_votes)
async def get_poll_registrations(self, request):
private_key, public_key, user = await self._authorize(request)
election_id = request.match_info.get('electionId', '')
if election_id == '':
raise ApiBadRequest(
'The election ID is a required query string parameter')
poll_book = await self._database.fetch_poll_book(
election_id=election_id)
if poll_book is None:
raise ApiNotFound('No voters with the election id '
'{} was not found'.format(election_id))
election = await self._database.fetch_election_resource(
election_id=election_id)
if election is None:
raise ApiNotFound('Election with the election id '
'{} was not found'.format(election_id))
if election.get('admin_id') != user.get('voter_id'):
raise ApiBadRequest(
'User is not the owner of the election with the id '
'{} .'.format(election_id))
return json_response(poll_book)
async def count_poll_registrations(self, request):
private_key, public_key, user = await self._authorize(request)
election_id = request.match_info.get('electionId', '')
if election_id == '':
raise ApiBadRequest(
'The election ID is a required query string parameter')
count_poll_book = await self._database.count_poll_book(
election_id=election_id)
election = await self._database.fetch_election_resource(
election_id=election_id)
if election is None:
raise ApiNotFound('Election with the election id '
'{} was not found'.format(election_id))
if election.get('results_permission') != 'PUBLIC':
poll_registration = await self._database.fetch_poll_book_registration(
voter_id=user.get('voter_id'), election_id=election_id)
if poll_registration is None and election.get(
'admin_id') != user.get('voter_id'):
raise ApiBadRequest(
'Voter is not registered in the poll book of the election with the id '
'{} .'.format(election_id))
return json_response(count_poll_book)
async def list_voting_options_election(self, request):
private_key, public_key, user = await self._authorize(request)
election_id = request.match_info.get('electionId', '')
if election_id == '':
raise ApiBadRequest(
'The election ID is a required query string parameter')
voting_options = await self._database.fetch_election_voting_options_resource(
election_id=election_id)
if voting_options is None:
raise ApiNotFound('Voting Options in the election id '
'{} was not found'.format(election_id))
election = await self._database.fetch_election_resource(
election_id=election_id)
if election is None:
raise ApiNotFound('Election with the election id '
'{} was not found'.format(election_id))
if election.get('results_permission') != 'PUBLIC':
poll_registration = await self._database.fetch_poll_book_registration(
voter_id=user.get('voter_id'), election_id=election_id)
if poll_registration is None and election.get(
'admin_id') != user.get('voter_id'):
raise ApiForbidden(
'Voter is not registered in the poll book of the election with the id '
'{} .'.format(election_id))
return json_response(voting_options)
async def get_voting_option(self, request):
private_key, public_key, user = await self._authorize(request)
voting_option_id = request.match_info.get('votingOptionId', '')
if voting_option_id == '':
raise ApiBadRequest(
'The voting option ID is a required query string parameter')
voting_option = await self._database.fetch_voting_option_resource(
voting_option_id=voting_option_id)
if voting_option is None:
raise ApiNotFound('No voting options with the id '
'{} was not found'.format(voting_option_id))
return json_response(voting_option)
async def update_voting_option_status(self, request):
private_key, public_key, user = await self._authorize(request)
voting_option_id = request.match_info.get('votingOptionId', '')
if voting_option_id == '':
raise ApiBadRequest(
'The voting option ID is a required query string parameter')
voting_option = await self._database.fetch_voting_option_resource(
voting_option_id=voting_option_id)
if voting_option is None:
raise ApiNotFound('Voting Option with the voting option id '
'{} was not found'.format(voting_option_id))
election_id = voting_option.get('election_id')
election = await self._database.fetch_election_resource(
election_id=election_id)
if election is None:
raise ApiNotFound('Election with the election id '
'{} was not found'.format(election_id))
current_time = get_time()
if election.get('start_timestamp') < current_time:
raise ApiBadRequest('Election with the election id '
'{} already start.'.format(election_id))
if election.get('admin_id') != user.get('voter_id'):
raise ApiBadRequest(
'User is not the owner of the election with the id '
'{} .'.format(election_id))
if voting_option.get('status') is True:
status = 0
else:
status = 1
await self._messenger.send_update_voting_option_status_transaction(
private_key=private_key,
voting_option_id=voting_option_id,
name=voting_option.get('name'),
description=voting_option.get('description'),
election_id=voting_option.get('election_id'),
status=status,
timestamp=get_time())
return json_response(
{'data': 'Update Voting Option Status transaction submitted'})
async def update_poll_book_status(self, request):
private_key, public_key, user = await self._authorize(request)
voter_id = request.match_info.get('voterId', '')
if voter_id == '':
raise ApiBadRequest(
'The voter ID is a required query string parameter')
election_id = request.match_info.get('electionId', '')
if election_id == '':
raise ApiBadRequest(
'The election ID is a required query string parameter')
voter_poll_book = await self._database.fetch_poll_book_registration(
election_id=election_id, voter_id=voter_id)
if voter_poll_book is None:
raise ApiNotFound('Voter with the voter id '
'{} was not found'.format(voter_id))
election = await self._database.fetch_election_resource(
election_id=election_id)
if election is None:
raise ApiNotFound('Election with the election id '
'{} was not found'.format(election_id))
current_time = get_time()
if election.get('start_timestamp') < current_time:
raise ApiBadRequest('Election with the election id '
'{} already start.'.format(election_id))
if election.get('admin_id') != user.get('voter_id'):
raise ApiBadRequest(
'User is not the owner of the election with the id '
'{} .'.format(election_id))
if voter_poll_book.get('status') is True:
status = 0
else:
status = 1
await self._messenger.send_update_voter_poll_book_status_transaction(
private_key=private_key,
voter_id=voter_id,
name=voter_poll_book.get('name'),
election_id=election_id,
status=status,
timestamp=get_time())
return json_response(
{'data': 'Update Poll Registration Status transaction submitted'})
async def list_elections_current(self, request):
private_key, public_key, user = await self._authorize(request)
voterId = request.match_info.get('voterId', '')
if voterId == '':
raise ApiBadRequest(
'The voter ID is a required query string parameter')
if user.get('voter_id') != voterId:
raise ApiForbidden('Admin must be the authenticated one')
current_elections_list = await self._database.fetch_current_elections_resources(
voterId, get_time())
return json_response(current_elections_list)
async def list_elections_past(self, request):
private_key, public_key, user = await self._authorize(request)
voterId = request.match_info.get('voterId', '')
if voterId == '':
raise ApiBadRequest(
'The voter ID is a required query string parameter')
if user.get('voter_id') != voterId:
raise ApiForbidden('Admin must be the authenticated one')
past_elections_list = await self._database.fetch_past_elections_resources(
voterId, get_time())
return json_response(past_elections_list)
async def list_admin_elections(self, request):
private_key, public_key, user = await self._authorize(request)
voter_id = request.match_info.get('voterId', '')
if voter_id == '':
raise ApiBadRequest(
'The voter ID is a required query string parameter')
if user.get('voter_id') != voter_id:
raise ApiForbidden('Admin must be the authenticated one')
if user.get('type') != 'ADMIN' and user.get('type') != 'SUPERADMIN':
raise ApiForbidden('Voter must be an admin or superadmin')
admin_elections_list = await self._database.fetch_admin_elections_resources(
user.get('voter_id'))
return json_response(admin_elections_list)
async def list_public_elections(self, request):
private_key, public_key, user = await self._authorize(request)
public_elections_list = await self._database.fetch_public_elections_resources(
get_time())
return json_response(public_elections_list)
async def list_public_past_elections(self, request):
private_key, public_key, user = await self._authorize(request)
past_elections_list = await self._database.fetch_public_past_elections_resources(
user.get('voter_id'), get_time())
return json_response(past_elections_list)
async def list_admins(self, request):
private_key, public_key, user = await self._authorize(request)
if user.get('type') != 'SUPERADMIN':
raise ApiForbidden('Forbidden')
admin_list = await self._database.fetch_admins_resources()
return json_response(admin_list)
async def get_voters(self, request):
private_key, public_key, user = await self._authorize(request)
if user.get('type') != 'SUPERADMIN':
raise ApiForbidden('Forbidden')
voter_id = request.match_info.get('voterId', '')
voters_list = await self._database.fetch_voters_resources(
voter_id=voter_id)
return json_response(voters_list)
async def get_vote(self, request):
private_key, public_key, user = await self._authorize(request)
vote_id = request.match_info.get('voteId', '')
if vote_id == '':
raise ApiBadRequest(
'The vote ID is a required query string parameter')
vote = await self._database.fetch_vote_resource(vote_id=vote_id)
if vote is None:
raise ApiNotFound('Vote with the vote id '
'{} was not found'.format(vote_id))
election_id = vote.get('election_id')
election = await self._database.fetch_election_resource(
election_id=election_id)
if election is None:
raise ApiNotFound('Election with the election id '
'{} was not found'.format(election_id))
return json_response(vote)
async def get_vote_election(self, request):
private_key, public_key, user = await self._authorize(request)
voter_id = request.match_info.get('voterId', '')
if voter_id == '':
raise ApiBadRequest(
'The voter ID is a required query string parameter')
election_id = request.match_info.get('electionId', '')
if election_id == '':
raise ApiBadRequest(
'The election ID is a required query string parameter')
if user.get('voter_id') != voter_id:
raise ApiForbidden('Admin must be the authenticated one')
vote = await self._database.fetch_my_vote__election_resource(
voter_id=voter_id, election_id=election_id)
return json_response(vote)
async def authenticate(self, request):
body = await decode_request(request)
required_fields = ['voter_id', 'password']
validate_fields(required_fields, body)
password = bytes(body.get('password'), 'utf-8')
voter = await self._database.fetch_voter_resource(
voter_id=body.get('voter_id'))
if voter is None:
raise ApiUnauthorized('Incorrect voter_id or password')
auth_info = await self._database.fetch_auth_resource(
public_key=voter.get('public_key'))
if auth_info is None:
raise ApiUnauthorized('No voter with that public key exists')
hashed_password = auth_info.get('hashed_password')
if not bcrypt.checkpw(password, bytes.fromhex(hashed_password)):
raise ApiUnauthorized('Incorrect public key or password')
user = {
'name': voter.get('name'),
'voter_id': body.get('voter_id'),
'type': voter.get('type')
}
token = self.generate_auth_token(request.app['secret_key'],
voter.get('public_key'), user)
return json_response({'accessToken': token, 'user': user})
async def _authorize(self, request):
token = request.headers.get('AUTHORIZATION')
if token is None:
raise ApiUnauthorized('No auth token provided')
token_prefixes = ('Bearer', 'Token')
for prefix in token_prefixes:
if prefix in token:
token = token.partition(prefix)[2].strip()
try:
token_dict = self.deserialize_auth_token(request.app['secret_key'],
token)
except BadSignature:
raise ApiUnauthorized('Invalid auth token')
public_key = token_dict.get('public_key')
auth_resource = await self._database.fetch_auth_resource(
public_key=public_key)
if auth_resource is None:
raise ApiUnauthorized('Token is not associated with an agent')
user = self.cache.get(token)
return decrypt_private_key(
request.app['aes_key'], public_key,
auth_resource['encrypted_private_key']), public_key, user
async def logout(self, request):
await self._authorize(request)
token = request.headers.get('AUTHORIZATION')
token_prefixes = ('Bearer', 'Token')
for prefix in token_prefixes:
if prefix in token:
token = token.partition(prefix)[2].strip()
self.cache.delete(token)
return json_response("Successful logout")
def generate_auth_token(self, secret_key, public_key, user):
serializer = Serializer(secret_key, expires_in=3600)
token = serializer.dumps({'public_key': public_key})
decoded_token = token.decode('ascii')
self.cache.set(decoded_token, user, expire=3600)
self.cache.close()
return decoded_token
def deserialize_auth_token(self, secret_key, token):
token_status = self.cache.get(token)
if token_status is None:
raise BadSignature("")
serializer = Serializer(secret_key)
return serializer.loads(token)
class OptimizationDriver():
"""
Object to interface the HOPP design analysis methods (i.e., sampling and optimization)
.. TODO: We might want to rename this to ProblemDriver? As it is required for non-optimization runs.
Parameters
----------
time_limit : initializer(float), optional
Total time limit in seconds
eval_limit : initializer(float), optional
Objective evaluation limit (counts new evaluations only)
obj_limit : initializer(float), optional
Lower bound of objective, exit if best objective is less than this
n_proc : initializer(int), optional
Maximum number of objective process workers
cach_dir : initializer(str), optional
Filename for the driver cache file
reconnect_cache : initializer(bool), optional
``True`` if the driver should reconnect to a previous result cache
write_csv : initializer(bool), optional
``True`` if the cached results should be written to csv format files
dataframe_file : initializer(str), optional
Filename for the driver cache dataframe file
csv_file : initializer(str), optional
filename for the driver cache csv file (if ``write_csv`` is ``True``)
scaled : initializer(bool), optional
``True`` if the sample/optimizer candidates need to be scaled to problem units
retry : initializer(bool), optional
``True`` if any evaluations ending in an exception should be retried on restart
"""
DEFAULT_KWARGS = dict(
time_limit=np.inf, # total time limit in seconds
eval_limit=np.
inf, # objective evaluation limit (counts new evaluations only)
obj_limit=-np.
inf, # lower bound of objective, exit if best objective is less than this
n_proc=multiprocessing.cpu_count() -
4, # maximum number of objective process workers
cache_dir='driver_cache', # filename for the driver cache file
reconnect_cache=
False, # True if the driver should reconnect to a previous result cache
write_csv=
False, # True if the cached results should be written to csv format files
dataframe_file=
'study_results.df.gz', # filename for the driver cache dataframe file
csv_file='study_results.csv', # filename for the driver cache csv file
scaled=
True, # True if the sample/optimizer candidates need to be scaled to problem units
retry=True
) # True if any evaluations ending in an exception should be retried on restart
def __init__(self, setup: Callable, **kwargs) -> None:
"""
Object to interface the HOPP optimization problem with humpday optimizers
:param setup: Function which creates and returns a new instance of the optimization problem
:param kwargs: Optional keyword arguments to change driver options (see DEFAULT_KWARGS)
"""
self.setup = setup
self.problem = setup(
) # The driver needs an instance of the problem to access problem.candidate_from()
self.parse_kwargs(kwargs)
self.best_obj = None
self.cache_info = dict(hits=0, misses=0, size=0, total_evals=0)
self.meta = dict()
self.get_candidate = self.problem.candidate_from_unit_array if self.options['scaled'] \
else self.problem.candidate_from_array # Function to create formatted design candidates
self.start_time = None
self.force_stop = False
self.eval_count = 0
if not self.options['reconnect_cache']:
i = 1
check_dir = self.options['cache_dir']
while os.path.isdir(check_dir):
check_dir = f"{self.options['cache_dir']}_{i}"
i += 1
self.options['cache_dir'] = check_dir
self.cache = Cache(self.options['cache_dir'],
disk=JSONDisk,
disk_compress_level=9,
cull_limit=0,
statistics=1,
eviction_policy='none')
self.start_len = len(self.cache)
self.read_cache()
def parse_kwargs(self, kwargs: dict) -> None:
"""
Helper function to set defaults and update options with user-provided input
:param kwargs: Using ``**kwargs`` this is a dict of keyword arguments provided by the user
:return: None
"""
self.options = self.DEFAULT_KWARGS.copy()
for key, value in kwargs.items():
if key in self.options:
self.options[key] = value
else:
print(f"Ignoring unknown driver option {key}={value}")
def init_parallel_workers(self, num_workers: int) -> None:
"""
Create the communication queue, cache dictionary, thread lock, and worker processes
:param num_workers: Number of process-independent workers, which evaluate the objective.
:return:
"""
if not hasattr(self, 'tasks'):
self.tasks = multiprocessing.JoinableQueue()
self.lock = threading.Lock()
print(f"Creating {num_workers} workers")
self.workers = [
Worker(self.tasks, self.cache, self.setup)
for _ in range(num_workers)
]
# Start the workers polling the task queue
for w in self.workers:
w.start()
def cleanup_parallel(self) -> None:
"""
Cleanup all worker processes, signal them to exit cleanly, mark any pending tasks as complete
:return: None
"""
# If the driver receives a KeyboardInterrupt then the task queue needs to be emptied
if self.force_stop:
try:
# Mark all tasks complete
while True:
self.tasks.get(block=False)
self.tasks.task_done()
# Occurs when task queue is empty
except queue.Empty:
pass
else:
# Exit normally, None task signals each worker to exit
for i in range(len(self.workers)):
self.tasks.put((None, 'worker exit'))
# Wait for all of the tasks to finish
self.tasks.join()
for w in self.workers:
w.join()
del w
def check_interrupt(self) -> None:
"""
Check optional stopping criteria, these are specified by the user in the driver options
:return: None
"""
if self.force_stop:
raise OptimizerInterrupt
elapsed = time.time() - self.start_time
if elapsed > self.options['time_limit']:
# print(f"Driver exiting, time limit: {self.options['time_limit']} secs")
# logging.info(f"Driver exiting, time limit: {self.options['time_limit']} secs")
raise OptimizerInterrupt
if self.eval_count >= self.options['eval_limit']:
# print(f"Driver exiting, eval limit: {self.options['eval_limit']}")
# logging.info(f"Driver exiting, eval limit: {self.options['eval_limit']}")
raise OptimizerInterrupt
if (self.best_obj
is not None) and (self.best_obj <= self.options['obj_limit']):
# print(f"Driver exiting, obj limit: {self.options['obj_limit']}")
# logging.info(f"Driver exiting, obj limit: {self.options['obj_limit']}")
raise OptimizerInterrupt
def print_log_header(self) -> None:
"""
Print a linear solver-style log header.
"""
self.log_headers = ['N_Evals', 'Objective', 'Eval_Time', 'Total_Time']
self.log_widths = [len(header) + 5 for header in self.log_headers]
print()
print("##### HOPP Optimization Driver #####".center(
sum(self.log_widths)))
print("Driver Options:", self.options, sep="\n\t")
print("Optimizer Options:", self.opt_names, sep="\n\t")
print()
print("".join(
(val.rjust(width)
for val, width in zip(self.log_headers, self.log_widths))))
def print_log_line(self, reason, obj, eval_time) -> None:
"""
Print a linear solver-style log line.
:param info: Dictionary containing at least the evaluation time of the last iteration and reason why a log
line is being printed. Originally lines would be printed for a hit on the cache (denoted by a ``c`` prefix on the
the line, but this was removed, and lines are now only printed on new evaluations for conciseness.
:returns: None
"""
prefix_reasons = {'cache_hit': 'c ', 'new_best': '* ', '': ''}
prefix = prefix_reasons[reason]
best_objective_str = f"{obj:8g}" if obj is not None else "NA"
curr_time = time.time()
log_values = [
prefix + str(self.eval_count), f"{best_objective_str}",
f"{eval_time / 60:.2f} min",
f"{(curr_time - self.start_time) / 60:.2f} min"
]
print("".join((val.rjust(width)
for val, width in zip(log_values, self.log_widths))))
def print_log_end(self, best_candidate, best_objective):
candidate_str = str(best_candidate) \
.replace('[', '[\n ', 1) \
.replace('], ', '],\n ') \
.replace(']]', ']\n ]')
print()
print(f"Best Objective: {best_objective:.2f}")
print(f"Best Candidate:\n {candidate_str}")
def write_cache(self, pd_filename=None, csv_filename=None) -> None:
"""
Write driver cache out to pickle file
:param filename: Optional path of file to write out the cache to
:return: None
"""
# if self.start_len == len(self.cache) - 1:
# print(f"no new entries in cache ({len(self.cache) - 1} results), skipping write...")
# return
dt_string = datetime.now().strftime('%Y-%m-%d_%H.%M.%S')
pd_dir = os.path.join(self.options['cache_dir'], '_dataframe',
dt_string)
if not os.path.isdir(pd_dir):
os.makedirs(pd_dir)
if pd_filename is None:
pd_filename = os.path.join(pd_dir, self.options['dataframe_file'])
else:
pd_filename = os.path.join(pd_dir, pd_filename)
if self.options['write_csv'] or csv_filename is not None:
csv_dir = os.path.join(self.options['cache_dir'], '_csv',
dt_string)
if not os.path.isdir(csv_dir):
os.makedirs(csv_dir)
if csv_filename is None:
csv_filename = os.path.join(csv_dir, self.options['csv_file'])
else:
csv_filename = os.path.join(csv_dir, csv_filename)
# Gather meta data
self.meta['cache_info'] = self.cache_info.copy()
self.meta['driver_options'] = self.options
self.meta['start_time'] = self.start_time
self.meta['candidate_fields'] = self.problem.candidate_fields
self.meta['design_variables'] = self.problem.design_variables
self.meta['fixed_variables'] = self.problem.fixed_variables
try:
self.meta['problem_setup'] = inspect.getsource(self.setup)
except TypeError:
if isinstance(self.setup, partial):
self.meta['problem_setup'] = inspect.getsource(self.setup.func) \
+ '__args__' + str(self.setup.args) + '\n' \
+ '__keywords__' + str(self.setup.keywords) + '\n'
try:
self.meta['sim_setup'] = inspect.getsource(
self.problem.init_simulation)
except TypeError:
if isinstance(self.problem.init_simulation, partial):
self.meta['sim_setup'] = inspect.getsource(self.problem.init_simulation.func) \
+ '__args__' + str(self.problem.init_simulation.args) + '\n' \
+ '__keywords__' + str(self.problem.init_simulation.keywords) + '\n'
try:
self.meta['eval_obj'] = inspect.getsource(
self.problem.evaluate_objective)
except TypeError:
if isinstance(self.problem.evaluate_objective, partial):
self.meta['eval_obj'] = inspect.getsource(self.problem.evaluate_objective.func) \
+ '__args__' + str(self.problem.evaluate_objective.args) + '\n' \
+ '__keywords__' + str(self.problem.evaluate_objective.keywords) + '\n'
self.cache['meta'] = self.meta.copy()
self.start_len = len(self.cache) - 1
print(
f"writing {len(self.cache) - 1} results to dataframe {pd_filename}..."
)
data_list = []
candidate_sep = self.problem.sep
pandas_sep = '__'
for candidate in self.cache:
if candidate == 'meta':
continue
result = self.cache.get(candidate)
if not isinstance(result, dict):
self.cache.delete(candidate)
continue
# row = dict()
#
# for key, value in candidate:
# key = key.replace(candidate_sep, pandas_sep)
# row[key] = value
#
# row.update(flatten_dict(result))
data_list.append(result)
df = pd.DataFrame(data_list)
df.attrs = self.meta
df.to_pickle(pd_filename)
### new code to write to csv(s)
if csv_filename is not None:
all_cols = sorted(df.columns)
scalar_cols = [
col for col in all_cols
if isinstance(df[col].loc[0], np.float64)
]
iterable_cols = [
col for col in all_cols
if not isinstance(df[col].loc[0], np.float64)
]
df[scalar_cols].to_csv(csv_filename)
for i in range(len(df)):
df_row = pd.concat(
[pd.Series(df[col].loc[i]) for col in iterable_cols],
axis=1,
keys=iterable_cols)
row_filename = os.path.join(csv_dir, f"{i}.csv")
df_row.to_csv(row_filename)
def read_cache(self) -> None:
"""
Read the driver cache from file
:param filename: Optional path of file to read the cache from
:return: None
"""
if len(self.cache) > 0:
try:
self.cache_info.update(self.cache['meta']['cache_info'])
self.meta.update(self.cache['meta'].copy())
except KeyError:
pass
return
def wrapped_parallel_objective(self):
"""
This method implements the logic to check if a candidate is in the cache, or is pending evaluation, or neither.
Each optimizer thread needs its own copy of this method since they don't have access to the driver object, we
can implement this by wrapping this method and returning the wrapped function. This allows the optimizer threads
to share the driver object without explicitly passing it to them, and allows them to all use the shared task
queue and driver cache.
:return: None
"""
eval_count = 0
@wraps(self.wrapped_parallel_objective)
def p_wrapper(*args, name=None, idx=None, objective=None):
"""
Objective function the optimizer threads call, assumes a parallel structure and avoids any re-calculations
- Check if candidate is in cache, if so return objective stored in cache
- If not, check if candidate is in queue (indicated by integer value in cache), wait for signal
- If not, objective needs to be calculated, add candidate to task queue, poll cache for return,
and finally signal any threads waiting on the same candidate
:param args: Follows the optimizer's convention of objective inputs (typically an array of floats)
:param name: Caller name to insert into the result dictionary
:param idx: Thread index, used for signal conditions
:param objective_keys: Ordered list of keys to get the objective from the result dictionary
:return: the numeric value being optimized
"""
nonlocal eval_count
eval_count += 1
self.check_interrupt()
candidate = self.get_candidate(*args)
self.cache_info['total_evals'] += 1
obj = None
try:
# Check if result in cache, throws KeyError if not
self.lock.acquire()
result = self.cache[candidate]
# print(f"cache hit {self.cache_info['total_evals']}")
self.lock.release()
self.cache_info['hits'] += 1
if isinstance(result, int):
# In cache but not complete, wait for complete signal
signal = self.conditions[result]
with signal:
signal.wait()
result = self.cache[candidate]
if not isinstance(result, dict):
self.force_stop = True
self.check_interrupt()
if 'exception' in result.keys():
if self.options['retry']:
with self.lock:
self.cache.delete(candidate)
raise KeyError
# Result available in cache, no work needed
# Append this caller name to the result dictionary
with self.lock:
result['caller'].append((name, eval_count))
self.cache[candidate] = result
except KeyError:
# Candidate not in cache, nor waiting in queue
self.cache[
candidate] = idx # indicates waiting condition for any other thread
# Insert candidate and caller information into task queue
self.tasks.put((candidate, (name, eval_count)))
self.lock.release()
self.cache_info['misses'] += 1
# Poll cache for available result (unclear how this could be a threading.Condition signal)
result = self.cache[candidate]
while isinstance(result, int):
time.sleep(10)
result = self.cache[candidate]
# Signal any other threads waiting on the same candidate
signal = self.conditions[idx]
with signal:
signal.notifyAll()
# KeyboardInterrupt places a OptimizerInterrupt in the cache to signal a force_stop
if not isinstance(result, dict):
self.force_stop = True
self.check_interrupt()
# Update best best objective if needed, and print a log line to console
if objective is not None:
obj = objective(result)
if (self.best_obj is None) or (obj < self.best_obj):
self.best_obj = obj
reason = 'new_best'
else:
reason = ''
else:
reason = ''
with self.lock:
self.eval_count += 1
self.print_log_line(reason, obj, result['eval_time'])
self.cache_info['size'] += 1
return obj
return p_wrapper
def wrapped_objective(self):
"""
This method implements the logic to check if a candidate is in the cache, or is pending evaluation, or neither.
Each optimizer thread needs its own copy of this method since they don't have access to the driver object, we
can implement this by wrapping this method and returning the wrapped function. This allows the optimizer threads
to share the driver object without explicitly passing it to them, and allows them to all use the shared task
queue and driver cache.
:return: None
"""
eval_count = 0
@wraps(self.wrapped_objective)
def s_wrapper(*args, name=None, objective=None):
"""
Objective function the optimizer threads call, assumes a parallel structure and avoids any re-calculations
- Check if candidate is in cache, if so return objective stored in cache
- If not, check if candidate is in queue (indicated by integer value in cache), wait for signal
- If not, objective needs to be calculated, add candidate to task queue, poll cache for return,
and finally signal any threads waiting on the same candidate
:param args: Follows the optimizer's convention of objective inputs (typically an array of floats)
:param name: Caller name to insert into the result dictionary
:param idx: Thread index, used for signal conditions
:param objective_keys: Ordered list of keys to get the objective from the result dictionary
:return: the numeric value being optimized
"""
nonlocal eval_count
eval_count += 1
self.check_interrupt()
candidate = self.get_candidate(*args)
self.cache_info['total_evals'] += 1
obj = None
try:
result = self.cache[candidate]
# print(f"cache hit {self.cache_info['total_evals']}")
if 'exception' in result.keys():
if self.options['retry']:
self.cache.delete(candidate)
raise KeyError
self.cache_info['hits'] += 1
# Result available in cache, no work needed
# Append this caller name to the result dictionary
result['caller'].append((name, eval_count))
self.cache[candidate] = result
except KeyError:
# Execute task, measure evaluation time
start_time = time.time()
result = self.problem.evaluate_objective(candidate)
result['eval_time'] = time.time() - start_time
result['caller'] = [(name, eval_count)]
self.cache[candidate] = result
self.cache_info['misses'] += 1
# KeyboardInterrupt places a OptimizerInterrupt in the cache to signal a force_stop
if not isinstance(result, dict):
self.force_stop = True
self.check_interrupt()
# Update best objective if needed, and print a log line to console
if objective is not None:
obj = objective(result)
if (self.best_obj is None) or (obj < self.best_obj):
self.best_obj = obj
reason = 'new_best'
else:
reason = ''
else:
reason = ''
self.eval_count += 1
self.print_log_line(reason, obj, result['eval_time'])
self.cache_info['size'] += 1
return obj
return s_wrapper
def execute(self, callables, inputs):
"""
Execute each pairwise callable given input in separate threads, using up to n_processors or the number of
callables whichever is less.
:param callables: A list of callable functions (e.g. a list of optimizer functions)
:param inputs: A list of inputs, one for each callable (e.g. a list of wrapped problem objectives)
:param objective_keys: A list of keys for the result nested dictionary structure
:return: Either the best objective found, corresponding to objective_keys, or the number of
successful evaluations if objective_keys is None
"""
# setup
self.start_time = time.time()
self.eval_count = 0
self.force_stop = False
# Begin parallel execution
self.print_log_header()
output = dict()
try:
for f, input, name in zip(callables, inputs, self.opt_names):
try:
output[name] = f(input)
# On an OptimizerInterrupt cancel all pending futures
except OptimizerInterrupt:
break
# Print any others
except Exception as exc:
err_str = traceback.format_exc()
print(f"{name} generated an exception: {err_str}")
# Optimizer thread exits normally
else:
# Print done message if input is a function (as in an optimization run)
if callable(inputs[0]):
print(f"Optimizer {name} finished")
# Allows clean exit on KeyboardInterrupt
except KeyboardInterrupt:
pass
self.write_cache()
self.cache.close()
return output
def parallel_execute(self, callables, inputs):
"""
Execute each pairwise callable given input in separate threads, using up to n_processors or the number of
callables whichever is less.
:param callables: A list of callable functions (e.g. a list of optimizer functions)
:param inputs: A list of inputs, one for each callable (e.g. a list of wrapped problem objectives)
:param objective_keys: A list of keys for the result nested dictionary structure
:return: Either the best objective found, corresponding to objective_keys, or the number of
successful evaluations if objective_keys is None
"""
# setup
self.start_time = time.time()
self.eval_count = 0
self.force_stop = False
# Establish communication queues and execution workers-
num_workers = min(
self.options['n_proc'],
len(callables)) # optimizers are assumed to be serial
self.init_parallel_workers(num_workers)
# Add thread conditions to allow signaling between threads waiting on the same candidate
self.conditions = [
threading.Condition() for _ in range(len(callables))
]
# Begin parallel execution
self.print_log_header()
output = dict()
with cf.ThreadPoolExecutor(max_workers=num_workers) as executor:
try:
threads = {
executor.submit(callables[i], inputs[i]): name
for i, name in enumerate(self.opt_names)
}
for future in cf.as_completed(threads):
name = threads[future]
try:
output[name] = future.result()
# On an OptimizerInterrupt cancel all pending futures
except OptimizerInterrupt:
for f, name in threads.items():
f.cancel()
break
# Print any others
except Exception as exc:
err_str = traceback.format_exc()
print(f"{name} generated an exception: {err_str}")
# Optimizer thread exits normally
else:
# Print done message if input is a function (as in an optimization run)
if callable(inputs[0]):
print(f"Optimizer {name} finished")
# Allows clean exit on KeyboardInterrupt
except KeyboardInterrupt:
for future, name in threads.items():
future.cancel()
# End worker processes
self.cleanup_parallel()
self.write_cache()
self.cache.close()
return output
def sample(self, candidates, design_name='Sample') -> int:
"""
Execute the objective function on each candidate in a sample in parallel, using yp to n_processors or the
number of candidates threads.
:param candidates: A list of unit arrays corresponding to the samples of a design.
:return: The number of successful evaluations.
"""
n_candidates = len(candidates)
self.opt_names = [f"{design_name}-{i}" for i in range(n_candidates)]
callables = [
partial(self.wrapped_objective(), name=name)
for i, name in enumerate(self.opt_names)
]
evaluations = self.execute(callables, candidates)
return evaluations
def parallel_sample(self, candidates, design_name='Sample') -> int:
"""
Execute the objective function on each candidate in a sample in parallel, using yp to n_processors or the
number of candidates threads.
:param candidates: A list of unit arrays corresponding to the samples of a design.
:return: The number of successful evaluations.
"""
n_candidates = len(candidates)
self.opt_names = [f"{design_name}-{i}" for i in range(n_candidates)]
callables = [
partial(self.wrapped_parallel_objective(), name=name, idx=i)
for i, name in enumerate(self.opt_names)
]
output = self.parallel_execute(callables, candidates)
return output
def optimize(self, optimizers, opt_configs, objectives) -> tuple:
"""
Execute the the list of optimizers on an instance of the wrapped objective function, using up to n_processors
or the number of optimizers.
:param optimizers: A list of optimization callable functions, taking the function to be optimized and config.
:param opt_config: The common optimizer configuration, shared between all optimization functions.
:param objective_keys: A list of keys for the result nested dictionary structure
:return: The best candidate and best simulation result found.
"""
n_opt = len(optimizers)
self.opt_names = [
f"{opt.__name__}-{obj.__name__}"
for opt, obj in zip(optimizers, objectives)
]
self.meta['obj'] = {
f"{obj.__name__}-{i}": inspect.getsource(obj)
for i, obj in enumerate(objectives)
}
# Defining optimizer thread callables and inputs
# The wrapped objective function is the input to the optimizer
callables = [
partial(opt, **config)
for opt, config in zip(optimizers, opt_configs)
]
inputs = [
partial(self.wrapped_objective(), name=name, objective=obj)
for i, (name, obj) in enumerate(zip(self.opt_names, objectives))
]
# Some optimizers need the threads to have a __name__ attribute, partial objects do not
for i in range(n_opt):
inputs[i].__name__ = self.opt_names[i]
# best_candidate, best_result = self.execute(callables, inputs, objective=objective)
output = self.execute(callables, inputs)
return output
def parallel_optimize(self, optimizers, opt_configs, objectives) -> tuple:
"""
Execute the the list of optimizers on an instance of the wrapped objective function, using up to n_processors
or the number of optimizers.
:param optimizers: A list of optimization callable functions, taking the function to be optimized and config.
:param opt_config: The common optimizer configuration, shared between all optimization functions.
:param objective_keys: A list of keys for the result nested dictionary structure
:return: The best candidate and best simulation result found.
"""
n_opt = len(optimizers)
self.opt_names = [
f"{opt.__name__}-{obj.__name__}"
for opt, obj in zip(optimizers, objectives)
]
self.meta['obj'] = {
f"{obj.__name__}-{i}": inspect.getsource(obj)
for i, obj in enumerate(objectives)
}
# Defining optimizer thread callables and inputs
# The wrapped objective function is the input to the optimizer
callables = [
partial(opt, **config)
for opt, config in zip(optimizers, opt_configs)
]
inputs = [
partial(self.wrapped_parallel_objective(),
name=name,
objective=obj,
idx=i)
for i, (name, obj) in enumerate(zip(self.opt_names, objectives))
]
# Some optimizers need the threads to have a __name__ attribute, partial objects do not
for i in range(n_opt):
inputs[i].__name__ = self.opt_names[i]
# best_candidate, best_result = self.parallel_execute(callables, inputs, objective=objective)
output = self.parallel_execute(callables, inputs)
return output
