def setUp(self):
super(TestExperimentAPI, self).setUp()
added = self.experiment_mongo.add_participant_experiment(Auth.TESTER_EMAIL, 'test', 'control')
self.assertTrue(added)
added = self.experiment_mongo.add_participant_experiment(Auth.TESTER_EMAIL, 'test2', 'group2_a')
self.assertTrue(added)
self.exp_name_test_hash = Experiment._hash_string('test')
self.exp_group_test_hash = Experiment._hash_string('control')
self.exp_name_test2_hash = Experiment._hash_string('test2')
self.exp_group_test2_hash = Experiment._hash_string('group2_a')
def default_systematicity(chars, fonts, font_sizes):
for font in fonts:
for font_size in font_sizes:
with data.db.atomic():
experiment_name = "Default: {0} size {1}.".format(
font.name, font_size)
experiment = Experiment(
name=experiment_name,
method=ExperimentType.DefaultSystematicity,
start_time=datetime.now(),
hyperparameters=None)
experiment.save()
print(experiment_name)
renderer = shapes.GlyphRenderer(io.BytesIO(font.font_file))
defaults = [axis.default for axis in renderer._axes]
try:
result = systematicity.evaluate(chars,
font,
font_size,
coords=None)
except systematicity.FailedRenderException:
print(
"Unable to determine systematicity for {0} pt {1} because at least one glyph failed to render."
.format(font_size, font.name))
continue
save_result(experiment.id, result)
print("Corr: {0:.4f} for {1} pt {2}.".format(
result.edit_correlation, font_size, font.name))
experiment.end_time = datetime.now()
experiment.save()
def grid_search(chars, fonts, font_sizes, grid_count):
for font in fonts:
for font_size in font_sizes:
experiment_name = "Grid: {0} size {1}, {2} facets.".format(
font.name, font_size, grid_count)
experiment = Experiment(name=experiment_name,
method=ExperimentType.GridSearch,
start_time=datetime.now(),
hyperparameters=json.dumps(
{"facets": grid_count}))
experiment.save()
print(experiment_name)
random.seed(random_seed)
renderer = shapes.GlyphRenderer(io.BytesIO(font.font_file))
defaults = [axis.default for axis in renderer._axes]
best_corr = 0.0
for index in range(len(renderer._axes)):
axis = renderer._axes[index]
vals = get_grid_coords(axis.minimum, axis.maximum, grid_count)
best_axis_corr = 0.0
for idx, val in enumerate(vals):
coords = defaults.copy()
coords[index] = val
try:
result = systematicity.evaluate(
chars, font, font_size, coords)
except systematicity.FailedRenderException:
# ignore failed render and carry on
print("Failed render at point {0}".format(coords))
continue
save_result(experiment.id, result)
print("Corr {0:.4f} for {1} pt {2} for {3} value of {4}".
format(result.edit_correlation, font_size, font.name,
axis.name, val))
if result.edit_correlation > best_axis_corr:
best_axis_corr = result.edit_correlation
if result.edit_correlation > best_corr:
best_corr = result.edit_correlation
print("Best corr: {0:.4f} for axis {1}".format(
best_axis_corr, axis.name))
print("Best corr: {0:.4f}".format(best_corr))
experiment.end_time = datetime.now()
experiment.save()
def get(self, uuid):
participant = self.experiment.get_participant(uuid, 'uuid')
if participant:
p_exp = {'uuid': uuid}
if 'experiments' in participant.keys():
p_exp['experiments'] = Experiment.get_experiments_hash(
participant['experiments'])
else:
p_exp['experiments'] = {}
return p_exp
else:
msg = '404: User {} not found!'.format(uuid)
err = 404
logging.error('[API:UserExperiment] ERROR: {}'.format(msg))
return {'MESSAGE': msg}, err
def random_search(chars, fonts, font_sizes, num_points):
for font in fonts:
for font_size in font_sizes:
experiment_name = "Random: {0} size {1}, {2} points.".format(
font.name, font_size, num_points)
experiment = Experiment(name=experiment_name,
method=ExperimentType.RandomSearch,
start_time=datetime.now(),
hyperparameters=json.dumps(
{"points": num_points}))
experiment.save()
print(experiment_name)
random.seed(random_seed)
renderer = shapes.GlyphRenderer(io.BytesIO(font.font_file))
points = get_random_coords(renderer._axes, num_points)
# Include min and max
points.insert(0, [axis.minimum for axis in renderer._axes])
points.append([axis.maximum for axis in renderer._axes])
best_corr = 0.0
iteration = 1
for point in points:
try:
result = systematicity.evaluate(chars, font, font_size,
point)
except systematicity.FailedRenderException:
# ignore failed render and carry on to next point
print("{0} Failed render at point {1}".format(
iteration, point))
continue
save_result(experiment.id, result)
print("{0} Corr: {1:.4f} for {2} pt {3} with coords {4}...".
format(iteration, result.edit_correlation, font_size,
font.name, point))
if result.edit_correlation > best_corr:
best_corr = result.edit_correlation
iteration += 1
print("Best corr: {0:.4f}".format(best_corr))
experiment.end_time = datetime.now()
experiment.save()
def __init__(self, settings): self.client = MongoClient(settings.MONGODB_URL) self.db = self.client[settings.MONGODB_NAME] self.experiment = self.db[COLLECTION_AUTH] self.data = Experiment(settings)
class ExperimentMongo:
def __init__(self, settings):
self.client = MongoClient(settings.MONGODB_URL)
self.db = self.client[settings.MONGODB_NAME]
self.experiment = self.db[COLLECTION_AUTH]
self.data = Experiment(settings)
def add_participant(self, email):
stako_users = self.db[COLLECTION_USERS]
if self.get_participant(email) == {}:
a_user = self.data.get_empty_user()
a_user['email'] = email
result = self.experiment.insert_one(a_user)
saved = type(result.inserted_id) is ObjectId
logging.info('[Mongo:SaveExperimentUser] Saved? {}'.format(saved))
if saved:
empty_user = StakoUser.get_empty_user()
empty_user['uuid'] = a_user['uuid']
result2 = stako_users.insert_one(empty_user)
saved2 = type(result2.inserted_id) is ObjectId
if saved2:
return a_user['uuid']
else:
self.experiment.delete_one({"_id": result.inserted_id})
return None
def _update_participant(self, participant):
result = self.experiment.update_one({'email': participant['email']}, {'$set': participant}, upsert=False)
if result.modified_count == 1:
return True
else:
return False
# Makes sure the role is part of this participant's roles
def add_participant_role(self, email, role):
p = self.get_participant(email)
result = False
if p:
if role in self.data.ROLES:
if role not in p['roles']:
p['roles'].append(role)
result = self._update_participant(p)
else:
result = True
return result
# Makes sure the role is NOT part of this participant's roles
def remove_participant_role(self, email, role):
p = self.get_participant(email)
result = False
if p:
if role in p['roles']:
p['roles'].remove(role)
result = self._update_participant(p)
else:
result = True
return result
# Ensures participant is part of experiment <exp_name> and in group <exp_group>
# <exp_name> and <exp_group> HAVE TO be in data.Experiment
def add_participant_experiment(self, email, exp_name, exp_group):
p = self.get_participant(email)
result = False
if p:
if 'experiments' not in p.keys():
p['experiments'] = {}
if exp_name in self.data.EXPERIMENTS.keys():
if exp_group in self.data.EXPERIMENTS[exp_name]:
p['experiments'][exp_name] = exp_group
result = self._update_participant(p)
return result
# Ensures participant is NOT part of experiment <exp_name>
# <exp_name> HAS TO be in data.Experiment
def remove_participant_experiment(self, email, exp_name):
p = self.get_participant(email)
result = False
if p:
if 'experiments' not in p.keys():
p['experiments'] = {}
if exp_name in p['experiments'].keys():
p['experiments'].pop(exp_name, None)
result = self._update_participant(p)
else:
result = True
return result
def get_participant(self, by_value, by_key='email'):
collection = self.db[COLLECTION_AUTH]
if isinstance(by_value, str):
by_value = by_value.lower()
a_user = collection.find_one({by_key: by_value}, {'_id': 0})
if a_user:
return loads(dumps(a_user))
else:
return {}
def get_all(self):
collection = self.db[COLLECTION_AUTH]
return collection.find({}, {'_id': 0})
def simulated_annealing(chars,
fonts,
font_sizes,
init_temp,
time,
alter_type="gaussian",
alter_range="0.1",
method=ExperimentType.SimulatedAnnealing):
if method not in [
ExperimentType.SimulatedAnnealing,
ExperimentType.SimulatedAnnealingMin
]:
raise ("Method must be one of the simulated annealing types")
for font in fonts:
for font_size in font_sizes:
experiment_name = "Simulated Annealing: {0} size {1}, initial temp {2}, {3} iterations.".format(
font.name, font_size, init_temp, time)
experiment = Experiment(name=experiment_name,
method=method,
start_time=datetime.now(),
hyperparameters=json.dumps({
"temp":
init_temp,
"iterations":
time,
"alteration_type":
alter_type,
"alteration_range":
alter_range
}))
experiment.save()
random.seed(random_seed)
print(experiment_name)
temperature = init_temp
renderer = shapes.GlyphRenderer(io.BytesIO(font.font_file))
#candidate = get_random_coords(renderer._axes, 1)[0]
candidate = [axis.default for axis in renderer._axes]
result = systematicity.evaluate(chars, font, font_size, candidate)
save_result(experiment.id, result)
corr = result.edit_correlation
iteration = 1
best_candidate = candidate
best_corr = corr
best_iteration = iteration
print("Starting at {0}, {1}".format(best_candidate, best_corr))
while iteration < time and temperature > 0:
if alter_type == "gaussian":
new_candidate = alter_gaussian(candidate, renderer._axes,
alter_range)
else:
new_candidate = alter_uniform(candidate, renderer._axes,
alter_range)
try:
result = systematicity.evaluate(chars, font, font_size,
new_candidate)
except systematicity.FailedRenderException:
# ignore failed render and carry on to a new candidate
continue
save_result(experiment.id, result)
new_corr = result.edit_correlation
delta = new_corr - corr
if method == ExperimentType.SimulatedAnnealingMin:
delta = -(delta)
p = random.uniform(0.0, 1.0)
if math.exp(delta / temperature) > p:
print(
"{0:3d} MOVE: {1:.4f}, {2:.4f} > {3:.4f}, temp: {4:.4f}, {5}"
.format(iteration, new_corr,
math.exp(delta / temperature), p, temperature,
new_candidate))
candidate = new_candidate
corr = new_corr
else:
print(
"{0:3d} STAY: {1:.4f}, {2:.4f} <= {3:.4f}, temp: {4:.4f}, {5}"
.format(iteration, new_corr,
math.exp(delta / temperature), p, temperature,
new_candidate))
if ((method == ExperimentType.SimulatedAnnealing
and corr > best_corr)
or (method == ExperimentType.SimulatedAnnealingMin
and corr < best_corr)):
best_corr = corr
best_candidate = candidate
best_iteration = iteration
iteration += 1
temperature = init_temp * (1 - iteration / time)
print(
"Best candidate for {0} size {1} in iteration {2}: {3:.4f}, {4}"
.format(font.name, font_size, best_iteration, best_corr,
best_candidate))
experiment.end_time = datetime.now()
experiment.save()