def runOptimizationTask(task,
task_solver,
initial_distribution,
updater,
n_updates,
n_samples_per_update,
fig=None,
directory=None):
distribution = initial_distribution
all_costs = []
learning_curve = []
exploration_curve = []
if fig:
ax_space = fig.add_subplot(141)
ax_rollout = fig.add_subplot(142)
# Optimization loop
for i_update in range(n_updates):
# 0. Get cost of current distribution mean
cost_vars_eval = task_solver.performRollout(distribution.mean)
cost_eval = task.evaluateRollout(cost_vars_eval, distribution.mean)
rollout_eval = Rollout(distribution.mean, cost_vars_eval, cost_eval)
# 1. Sample from distribution
samples = distribution.generateSamples(n_samples_per_update)
# 2. Evaluate the samples
costs = np.full(n_samples_per_update, 0.0)
rollouts = []
for i_sample in range(n_samples_per_update):
# 2A. Perform the rollouts
cost_vars = task_solver.performRollout(samples[i_sample, :])
# 2B. Evaluate the rollouts
cur_costs = task.evaluateRollout(cost_vars, samples[i_sample, :])
costs[i_sample] = cur_costs[0]
rollouts.append(Rollout(samples[i_sample, :], cost_vars,
cur_costs))
# 3. Update parameters
distribution_new, weights = updater.updateDistribution(
distribution, samples, costs)
# Bookkeeping and plotting
# All the costs so far
all_costs.extend(costs)
# Update exploration curve
cur_samples = i_update * n_samples_per_update
cur_exploration = np.sqrt(distribution.maxEigenValue())
exploration_curve.append([cur_samples, cur_exploration])
# Update learning curve
learning_curve.append([cur_samples])
learning_curve[-1].extend(cost_eval)
# Plot summary of this update
if fig:
highlight = (i_update == 0)
plotUpdate(distribution, cost_eval, samples, costs, weights,
distribution_new, ax_space, highlight)
# Try plotting rollout with task solver
h = task_solver.plotRollout(rollout_eval.cost_vars, ax_rollout)
if not h:
# Task solver didn't know how to plot the rollout, try plotting
# it with task instead
h = task.plotRollout(rollout_eval.cost_vars, ax_rollout)
if h:
# If there is a handle, change its color depending on i_update
setColor(h, i_update, n_updates)
if directory:
saveUpdateRollouts(directory, i_update, distribution, rollout_eval,
rollouts, weights, distribution_new)
# Distribution is new distribution
distribution = distribution_new
# Plot learning curve
cost_labels = task.costLabels()
if fig:
plotExplorationCurve(exploration_curve, fig.add_subplot(143))
plotLearningCurve(learning_curve, fig.add_subplot(144), all_costs,
cost_labels)
# Save learning curve to file, if necessary
if directory:
saveLearningCurve(directory, learning_curve)
saveExplorationCurve(directory, exploration_curve)
print('Saved results to "' + directory + '".')
return learning_curve
lib_path = os.path.abspath('../../python/')
sys.path.append(lib_path)
from bbo.bbo_plotting import plotLearningCurve, plotExplorationCurve
if __name__=='__main__':
covar_updates = ["none","decay","adaptation"]
figure_number = 1;
for covar_update in covar_updates:
print("Run C++ optimization with covar update '"+covar_update+"'")
# Call the executable with the directory to which results should be written
executable = "../../bin/demoOptimization"
directory = "/tmp/demoOptimization/"+covar_update
executeBinary(executable,directory+" "+covar_update)
print(" Plotting results")
fig = plt.figure(figure_number)
figure_number += 1;
exploration_curve = np.loadtxt(directory+'/exploration_curve.txt')
plotExplorationCurve(exploration_curve,fig.add_subplot(121))
learning_curve = np.loadtxt(directory+'/learning_curve.txt')
plotLearningCurve(learning_curve,fig.add_subplot(122))
fig.canvas.set_window_title("Optimization with covar_update="+covar_update)
plt.show()
def runOptimizationTask(task, task_solver, initial_distribution, updater, n_updates, n_samples_per_update,fig=None,directory=None):
distribution = initial_distribution
all_costs = []
learning_curve = []
exploration_curve = []
if fig:
ax_space = fig.add_subplot(141)
ax_rollout = fig.add_subplot(142)
# Optimization loop
for i_update in range(n_updates):
# 0. Get cost of current distribution mean
cost_vars_eval = task_solver.performRollout(distribution.mean)
cost_eval = task.evaluateRollout(cost_vars_eval,distribution.mean)
rollout_eval = Rollout(distribution.mean,cost_vars_eval,cost_eval)
# 1. Sample from distribution
samples = distribution.generateSamples(n_samples_per_update)
# 2. Evaluate the samples
costs = np.full(n_samples_per_update,0.0)
rollouts = []
for i_sample in range(n_samples_per_update):
# 2A. Perform the rollouts
cost_vars = task_solver.performRollout(samples[i_sample,:])
# 2B. Evaluate the rollouts
cur_costs = task.evaluateRollout(cost_vars,samples[i_sample,:])
costs[i_sample] = cur_costs[0]
rollouts.append(Rollout(samples[i_sample,:],cost_vars,cur_costs))
# 3. Update parameters
distribution_new, weights = updater.updateDistribution(distribution, samples, costs)
# Bookkeeping and plotting
# All the costs so far
all_costs.extend(costs)
# Update exploration curve
cur_samples = i_update*n_samples_per_update
cur_exploration = np.sqrt(distribution.maxEigenValue())
exploration_curve.append([cur_samples,cur_exploration])
# Update learning curve
learning_curve.append([cur_samples])
learning_curve[-1].extend(cost_eval)
# Plot summary of this update
if fig:
highlight = (i_update==0)
plotUpdate(distribution,cost_eval,samples,costs,weights,distribution_new,ax_space,highlight)
# Try plotting rollout with task solver
h = task_solver.plotRollout(rollout_eval.cost_vars,ax_rollout)
if not h:
# Task solver didn't know how to plot the rollout, try plotting
# it with task instead
h = task.plotRollout(rollout_eval.cost_vars,ax_rollout)
if h:
# If there is a handle, change its color depending on i_update
setColor(h,i_update,n_updates)
if directory:
saveUpdateRollouts(directory, i_update, distribution, rollout_eval, rollouts, weights, distribution_new)
# Distribution is new distribution
distribution = distribution_new
# Plot learning curve
cost_labels = task.costLabels()
if fig:
plotExplorationCurve(exploration_curve,fig.add_subplot(143))
plotLearningCurve(learning_curve,fig.add_subplot(144),all_costs,cost_labels)
# Save learning curve to file, if necessary
if directory:
saveLearningCurve(directory,learning_curve)
saveExplorationCurve(directory,exploration_curve)
print('Saved results to "'+directory+'".')
return learning_curve
from executeBinary import executeBinary
lib_path = os.path.abspath('../../python/')
sys.path.append(lib_path)
from bbo.bbo_plotting import plotLearningCurve, plotExplorationCurve
if __name__ == '__main__':
covar_updates = ["none", "decay", "adaptation"]
figure_number = 1
for covar_update in covar_updates:
print("Run C++ optimization with covar update '" + covar_update + "'")
# Call the executable with the directory to which results should be written
executable = "../../bin/demoOptimization"
directory = "/tmp/demoOptimization/" + covar_update
executeBinary(executable, directory + " " + covar_update)
print(" Plotting results")
fig = plt.figure(figure_number)
figure_number += 1
exploration_curve = np.loadtxt(directory + '/exploration_curve.txt')
plotExplorationCurve(exploration_curve, fig.add_subplot(121))
learning_curve = np.loadtxt(directory + '/learning_curve.txt')
plotLearningCurve(learning_curve, fig.add_subplot(122))
fig.canvas.set_window_title("Optimization with covar_update=" +
covar_update)
plt.show()
def plotOptimizationRollouts(directory,fig,plotRollout=None,plot_all_rollouts=False):
if not fig:
fig = plt.figure(1,figsize=(9, 4))
# Determine number of updates
n_updates = 1
update_dir = '%s/update%05d' % (directory, n_updates)
while containsNewDistribution(update_dir):
n_updates += 1
update_dir = '%s/update%05d' % (directory, n_updates)
n_updates -= 1
if n_updates<2:
return None
all_costs = []
learning_curve = []
exploration_curve = []
i_samples = 0
for i_update in range(n_updates):
update_dir = '%s/update%05d' % (directory, i_update)
# Read data
mean = np.loadtxt(update_dir+"/distribution_mean.txt")
covar = np.loadtxt(update_dir+"/distribution_covar.txt")
distribution = DistributionGaussian(mean,covar)
try:
mean = np.loadtxt(update_dir+"/distribution_new_mean.txt")
covar = np.loadtxt(update_dir+"/distribution_new_covar.txt")
distribution_new = DistributionGaussian(mean,covar)
except IOError:
distribution_new = None
try:
covar_block_sizes = np.loadtxt(update_dir+"/covar_block_sizes.txt")
except IOError:
covar_block_sizes = len(distribution.mean)
try:
samples = np.loadtxt(update_dir+"/samples.txt")
except IOError:
samples = None
costs = np.loadtxt(update_dir+"/costs.txt")
weights = np.loadtxt(update_dir+"/weights.txt")
try:
cost_eval = np.loadtxt(update_dir+"/cost_eval.txt")
except IOError:
cost_eval = None
n_rollouts = len(weights)
rollouts = []
for i_rollout in range(n_rollouts):
rollout_dir = '%s/rollout%03d' % (update_dir, i_rollout+1)
rollouts.append(loadRolloutFromDirectory(rollout_dir))
rollout_eval = loadRolloutFromDirectory(update_dir+'/rollout_eval/')
# Update learning curve
# Bookkeeping and plotting
# All the costs so far
all_costs.extend(costs)
# Update exploration curve
i_samples = i_samples + n_rollouts
cur_exploration = np.sqrt(distribution.maxEigenValue())
exploration_curve.append([i_samples,cur_exploration])
# Update learning curve
learning_curve.append([i_samples])
learning_curve[-1].extend(np.atleast_1d(cost_eval))
n_subplots = 3
i_subplot = 1
if plotRollout:
n_subplots = 4
ax_rollout = fig.add_subplot(1,n_subplots,i_subplot)
i_subplot += 1
h = plotRollout(rollout_eval.cost_vars,ax_rollout)
setColor(h,i_update,n_updates)
ax_space = fig.add_subplot(1,n_subplots,i_subplot)
i_subplot += 1
highlight = (i_update==0)
plotUpdate(distribution,cost_eval,samples,costs,weights,distribution_new,ax_space,highlight)
plotExplorationCurve(exploration_curve,fig.add_subplot(1,n_subplots,i_subplot))
plotLearningCurve(learning_curve,fig.add_subplot(1,n_subplots,i_subplot+1))
def runOptimization(cost_function,
initial_distribution,
updater,
n_updates,
n_samples_per_update,
fig=None,
directory=None):
""" Run an evolutionary optimization process, see \ref page_bbo
\param[in] cost_function The cost function to optimize
\param[in] initial_distribution The initial parameter distribution
\param[in] updater The Updater used to update the parameters
\param[in] n_updates The number of updates to perform
\param[in] n_samples_per_update The number of samples per update
\param[in] fig Optional figure to plot the optimization.
\param[in] directory Optional directory to save to (default: don't save)
\return A learning curve that has the following format
#rows is number of optimization updates
column 0: Number of samples at which the cost was evaluated
column 1: The total cost
column 2...: Individual cost components (column 1 is their sum)
"""
distribution = initial_distribution
all_costs = []
learning_curve = []
exploration_curve = []
if fig:
ax = fig.add_subplot(131)
# Optimization loop
for i_update in range(n_updates):
# 0. Get cost of current distribution mean
cost_eval = cost_function.evaluate(distribution.mean)
# 1. Sample from distribution
samples = distribution.generateSamples(n_samples_per_update)
# 2. Evaluate the samples
costs = []
for i_sample in range(n_samples_per_update):
costs.append(cost_function.evaluate(samples[i_sample, :]))
# 3. Update parameters
distribution_new, weights = updater.updateDistribution(
distribution, samples, costs)
# Bookkeeping and plotting
# All the costs so far
all_costs.extend(costs)
# Update exploration curve
cur_samples = i_update * n_samples_per_update
cur_exploration = np.sqrt(distribution.maxEigenValue())
exploration_curve.append([cur_samples, cur_exploration])
# Update learning curve
learning_curve.append([cur_samples])
learning_curve[-1].extend(cost_eval)
# Plot summary of this update
if fig:
highlight = (i_update == 0)
plotUpdate(distribution, cost_eval, samples, costs, weights,
distribution_new, ax, highlight)
if directory:
saveUpdate(directory, i_update, distribution, cost_eval, samples,
costs, weights, distribution_new)
# Distribution is new distribution
distribution = distribution_new
# Plot learning curve
if fig:
plotExplorationCurve(exploration_curve, fig.add_subplot(132))
plotLearningCurve(learning_curve, fig.add_subplot(133), all_costs)
# Save learning curve to file, if necessary
if directory:
saveLearningCurve(directory, learning_curve)
saveExplorationCurve(directory, exploration_curve)
print('Saved results to "' + directory + '".')
return learning_curve
def runOptimization(cost_function,
initial_distribution,
updater,
n_updates,
n_samples_per_update,
fig=None,
directory=None):
distribution = initial_distribution
all_costs = []
learning_curve = []
exploration_curve = []
if fig:
ax = fig.add_subplot(131)
# Optimization loop
for i_update in range(n_updates):
# 0. Get cost of current distribution mean
cost_eval = cost_function.evaluate(distribution.mean)
# 1. Sample from distribution
samples = distribution.generateSamples(n_samples_per_update)
# 2. Evaluate the samples
costs = []
for i_sample in range(n_samples_per_update):
costs.append(cost_function.evaluate(samples[i_sample, :]))
# 3. Update parameters
distribution_new, weights = updater.updateDistribution(
distribution, samples, costs)
# Bookkeeping and plotting
# All the costs so far
all_costs.extend(costs)
# Update exploration curve
cur_samples = i_update * n_samples_per_update
cur_exploration = np.sqrt(distribution.maxEigenValue())
exploration_curve.append([cur_samples, cur_exploration])
# Update learning curve
learning_curve.append([cur_samples])
learning_curve[-1].extend(cost_eval)
# Plot summary of this update
if fig:
highlight = (i_update == 0)
plotUpdate(distribution, cost_eval, samples, costs, weights,
distribution_new, ax, highlight)
if directory:
saveUpdate(directory, i_update, distribution, cost_eval, samples,
costs, weights, distribution_new)
# Distribution is new distribution
distribution = distribution_new
# Plot learning curve
if fig:
plotExplorationCurve(exploration_curve, fig.add_subplot(132))
plotLearningCurve(learning_curve, fig.add_subplot(133), all_costs)
# Save learning curve to file, if necessary
if directory:
saveLearningCurve(directory, learning_curve)
saveExplorationCurve(directory, exploration_curve)
print('Saved results to "' + directory + '".')
return learning_curve