def find_accurate_policy(trials, alpha_0=10, beta_0=10, num_exs=250, students=None,
test_path=TEST_PATH, make_plot=False):
"""
Find the best alpha/beta for the teaching policy.
Args:
trials: Number of teaching plans to try out (including first plan).
alpha_0: Starting alpha.
beta_0: Starting beta.
num_exs: The number of examples given to students per teaching plan.
students: The students to teach, will default to STUDENTS if non given.
test_path: The path of the file with test qs/answers.
make_plot: Whether to make a scatter plot of the history.
Returns: The best alpha/beta found.
"""
if students is None:
students = STUDENTS
test_qs, test_ans = plan_eval.read_test(test_path)
history = []
eval_policy = _create_evaluator(num_exs, students, test_qs, test_ans, history)
experiment = Experiment([[0, ALPHA_MAX], [0, BETA_MAX]])
# Run the start experiment and evaluate.
experiment.historical_data.append_sample_points([eval_policy(alpha_0, beta_0)])
for i in xrange(trials-1):
print '--------TRIAL %d DONE--------' % (i + 1)
alpha, beta = gp_next_points(experiment)[0]
experiment.historical_data.append_sample_points([eval_policy(alpha, beta)])
best = max(history)
if make_plot:
plot_history(max(history), history)
return best
def find_fastest_policy(trials, alpha_0=10, beta_0=10, ex_cutoff=250, perf_thresh=0.93,
students=None, test_path=TEST_PATH, make_plot=True):
"""
Find the best alpha/beta for the teaching policy.
Args:
trials: Number of teaching plans to try out (including first plan).
alpha_0: Starting alpha.
beta_0: Starting beta.
ex_cutoff: Max number of examples to show.
perf_thresh: The threshold of what is considered perfect.
students: The students to teach, will default to STUDENTS if non given.
test_path: The path of the file with test qs/answers.
make_plot: Whether to make a scatter plot of the history.
Returns: The best alpha/beta found.
"""
if students is None:
students = STUDENTS
test_qs, test_ans = plan_eval.read_test(test_path)
history = []
eval_policy = _create_perf_evaluator(ex_cutoff, perf_thresh, students, test_qs, test_ans, history)
experiment = Experiment([[0, ALPHA_MAX], [0, BETA_MAX]])
# Run the start experiment and evaluate.
experiment.historical_data.append_sample_points([eval_policy(alpha_0, beta_0)])
for i in xrange(trials-1):
print '--------TRIAL %d DONE--------' % (i + 1)
alpha, beta = gp_next_points(experiment)[0]
experiment.historical_data.append_sample_points([eval_policy(alpha, beta)])
best = min(history)
print len(history)
print len(history)
if make_plot:
plot_history(min(history), history)
return best
def optimize_fastest(trials,
alpha_0=10,
beta_0=10,
ex_cutoff=25,
perf_thresh=0.8,
students=None,
test_path=TEST_PATH,
make_plot=True):
"""
Find the optimal teaching policy that will reach a threshold of performance
with the least amount of examples.
Args:
trials: The number of teaching plans to try out after init_points.
alpha_0: The first alpha to try.
beta_0: The first beta to try.
ex_cutoff: The maximum number of examples to try.
perf_thresh: The performance threshold to achieve.
students: List of student objects to teach where the average score
among students is considered.
test_path: The file path for the test to consider.
make_plot: Whether to make a plot of the results.
"""
if students is None:
students = STUDENTS
test_qs, test_ans = plan_eval.read_test(test_path)
def target_func(alpha, beta):
avg, var = plan_eval.teach_until_perfect(alpha, beta, students,
test_qs, test_ans, ex_cutoff,
perf_thresh)
# Wipe Students memory
for s in students:
s.wipe_memory()
return -1 * avg # Multiply by negative 1 to find min
# Do Bayes' optimization.
bo = BayesianOptimization(target_func, {
'alpha': (0, ALPHA_MAX),
'beta': (0, BETA_MAX)
})
bo.explore({'alpha': [alpha_0], 'beta': [beta_0]})
bo.maximize(init_points=5, n_iter=trials, acq='ucb', kappa=2)
max_val = bo.res['max']
best = (-1 * float(max_val['max_val']),
float(max_val['max_params']['alpha']),
float(max_val['max_params']['beta']))
if make_plot:
print bo.gp.predict(np.array([[10, 3], [10, 5]]))
alphas = [float(val['alpha']) for val in bo.res['all']['params']]
betas = [float(val['beta']) for val in bo.res['all']['params']]
vals = [-1 * float(v) for v in bo.res['all']['values']]
hist = [(vals[i], alphas[i], betas[i]) for i in xrange(len(alphas))]
plot_history(best, hist, bo)
return best
def optimize(trials,
alpha_0=10,
beta_0=10,
num_exs=15,
students=None,
test_path=TEST_PATH,
make_plot=True):
"""
Find the optimal teaching plan that maximizes student scores for a set
number of examples. Prints out optimal [alpha, beta] in terminal.
Args:
trials: The number of teaching plans to try out after init_points.
alpha_0: The first alpha to try.
beta_0: The first beta to try.
num_exs: The number of positive and negative examples to show students.
students: List of student objects to teach where the average score
among students is considered.
test_path: The file path for the test to consider.
make_plot: Whether to make a plot of the results.
"""
if students is None:
students = STUDENTS
test_qs, test_ans = plan_eval.read_test(test_path)
def target_func(alpha, beta):
avg, var = plan_eval.evaluate_plan(alpha, beta, students, num_exs,
test_qs, test_ans)
# Wipe Students memory
for s in students:
s.wipe_memory()
return avg
# Do Bayes' optimization.
bo = BayesianOptimization(target_func, {
'alpha': (0, ALPHA_MAX),
'beta': (0, BETA_MAX)
})
bo.explore({'alpha': [alpha_0], 'beta': [beta_0]})
bo.maximize(init_points=5, n_iter=trials, acq='ucb', kappa=2)
max_val = bo.res['max']
best = (float(max_val['max_val']), float(max_val['max_params']['alpha']),
float(max_val['max_params']['beta']))
if make_plot:
print bo.gp.predict(np.array([[10, 3], [10, 5]]))
alphas = [float(val['alpha']) for val in bo.res['all']['params']]
betas = [float(val['beta']) for val in bo.res['all']['params']]
vals = [float(v) for v in bo.res['all']['values']]
hist = [(vals[i], alphas[i], betas[i]) for i in xrange(len(alphas))]
plot_history(best, hist, bo)
return best
def _test_gen_read_test():
plan_eval.gen_unif_test(10, 'test.txt')
print plan_eval.read_test('test.txt')