def oneUpdate(directory,plot_results=False): inputs = np.linspace(-1.5,1.5,21); a = 2.0 c = -1.0 n_params = 2 regularization = 0.01 task = DemoTaskApproximateQuadraticFunction(a,c,inputs,regularization) mean_init = np.full(n_params,0.5) covar_init = 0.25*np.eye(n_params) initial_distribution = DistributionGaussian(mean_init, covar_init) eliteness = 10 weighting_method = 'PI-BB' covar_decay_factor = 0.8 updater = UpdaterCovarDecay(eliteness,weighting_method,covar_decay_factor) n_samples_per_update = 10 i_update = runOptimizationTaskOneUpdate(directory, task, initial_distribution, updater, n_samples_per_update) i_update -= 1 if plot_results and i_update>1: # Plot the optimization results (from the files saved to disk) fig = plt.figure(1,figsize=(15, 5)) plotOptimizationRollouts(directory,fig,task.plotRollout) plt.show()
def updateDistribution(self, distribution, samples, costs): """ Update a distribution with reward-weighted averaging. \param[in] distribution Distribution before the update \param[in] samples Samples in parameter space. \param[in] costs The cost of each sample. \return The updated distribution. """ mean_cur = distribution.mean covar_cur = distribution.covar n_samples = samples.shape[0] n_dims = samples.shape[1] weights = costsToWeights(costs, self.weighting_method, self.eliteness) # Compute new mean with reward-weighed averaging # mean = 1 x n_dims # weights = 1 x n_samples # samples = n_samples x n_dims mean_new = np.average(samples, 0, weights) #np.set_printoptions(precision=4, suppress=True) eps = samples - np.tile(mean_cur, (n_samples, 1)) weights_tiled = np.tile(np.asarray([weights]).transpose(), (1, n_dims)) weighted_eps = np.multiply(weights_tiled, eps) covar_new = np.dot(weighted_eps.transpose(), eps) # Remove non-diagonal values if (self.diag_only): diag_vec = np.diag(covar_new) covar_new = np.diag(diag_vec) # Low-pass filter for covariance matrix, i.e. weight between current # and new covariance matrix. if (self.learning_rate < 1.0): lr = self.learning_rate # For legibility covar_new = (1 - lr) * covar_cur + lr * covar_new # Add a base_level to avoid pre-mature convergence if (self.base_level_diagonal is not None): for ii in range(n_dims): if covar_new[ii, ii] < self.base_level_diagonal[ii]: covar_new[ii, ii] = self.base_level_diagonal[ii] # Update the covariance matrix distribution_new = DistributionGaussian(mean_new, covar_new) return distribution_new, weights
def updateDistribution(self, distribution, samples, costs): """ Update a distribution with reward-weighted averaging. \param[in] distribution Distribution before the update \param[in] samples Samples in parameter space. \param[in] costs The cost of each sample. \return The updated distribution. """ weights = costsToWeights(costs, self.weighting_method, self.eliteness) # Compute new mean with reward-weighed averaging # mean = 1 x n_dims # weights = 1 x n_samples # samples = n_samples x n_dims mean_new = np.average(samples, 0, weights) # Update the covariance matrix distribution_new = DistributionGaussian(mean_new, distribution.covar) return distribution_new, weights
if __name__ == "__main__": directory = None if (len(sys.argv) > 1): directory = sys.argv[1] n_dims = 2 minimum = np.full(n_dims, 2.0) regul_weight = 1.0 cost_function = DemoCostFunctionDistanceToPoint(minimum, regul_weight) mean_init = np.full(n_dims, 5.0) covar_init = 4.0 * np.eye(n_dims) distribution = DistributionGaussian(mean_init, covar_init) eliteness = 10 weighting_method = 'PI-BB' covar_decay_factor = 0.8 updater = UpdaterCovarDecay(eliteness, weighting_method, covar_decay_factor) n_samples_per_update = 20 n_updates = 40 import matplotlib.pyplot as plt fig = plt.figure(1, figsize=(15, 5)) learning_curve = runOptimization(cost_function, distribution, updater, n_updates, n_samples_per_update, fig,
# Normalize weights weights = weights/sum(weights) return weights if __name__=="__main__": eliteness = 10 weighting_method = 'PI-BB' covar_decay_factor = 0.8 updater = UpdaterCovarDecay(eliteness,weighting_method,covar_decay_factor) diagonal_min = 0.1 diagonal_max = 1.0 diag_only=False learning_rate=1.0 updater = UpdaterCovarAdaptation(eliteness, weighting_method,diagonal_max,diagonal_min,diag_only,learning_rate) mu = np.array([2,4]) cov = np.array([[0.3,0.0],[0.0,0.5]]) distribution = DistributionGaussian(mu,cov) n_samples = 10 samples = distribution.generateSamples(n_samples) costs = abs(samples[:,0]) + abs(samples[:,1]) # Manhattan distance (new_distribution, w) = updater.updateDistribution(distribution, samples, costs) print(distribution.covar) print(new_distribution.covar)
def plotOptimizationRollouts(directory, fig, plotRollout=None, plot_all_rollouts=False): if not fig: fig = plt.figure(1, figsize=(9, 4)) # Determine number of updates n_updates = 1 update_dir = '%s/update%05d' % (directory, n_updates) while containsNewDistribution(update_dir): n_updates += 1 update_dir = '%s/update%05d' % (directory, n_updates) n_updates -= 1 if n_updates < 2: return None all_costs = [] learning_curve = [] exploration_curve = [] i_samples = 0 for i_update in range(n_updates): update_dir = '%s/update%05d' % (directory, i_update) # Read data mean = np.loadtxt(update_dir + "/distribution_mean.txt") covar = np.loadtxt(update_dir + "/distribution_covar.txt") distribution = DistributionGaussian(mean, covar) try: mean = np.loadtxt(update_dir + "/distribution_new_mean.txt") covar = np.loadtxt(update_dir + "/distribution_new_covar.txt") distribution_new = DistributionGaussian(mean, covar) except IOError: distribution_new = None try: covar_block_sizes = np.loadtxt(update_dir + "/covar_block_sizes.txt") except IOError: covar_block_sizes = len(distribution.mean) try: samples = np.loadtxt(update_dir + "/samples.txt") except IOError: samples = None costs = np.loadtxt(update_dir + "/costs.txt") weights = np.loadtxt(update_dir + "/weights.txt") try: cost_eval = np.loadtxt(update_dir + "/cost_eval.txt") except IOError: cost_eval = None n_rollouts = len(weights) rollouts = [] for i_rollout in range(n_rollouts): rollout_dir = '%s/rollout%03d' % (update_dir, i_rollout + 1) rollouts.append(loadRolloutFromDirectory(rollout_dir)) rollout_eval = loadRolloutFromDirectory(update_dir + '/rollout_eval/') # Update learning curve # Bookkeeping and plotting # All the costs so far all_costs.extend(costs) # Update exploration curve i_samples = i_samples + n_rollouts cur_exploration = np.sqrt(distribution.maxEigenValue()) exploration_curve.append([i_samples, cur_exploration]) # Update learning curve learning_curve.append([i_samples]) learning_curve[-1].extend(np.atleast_1d(cost_eval)) n_subplots = 3 i_subplot = 1 if plotRollout: n_subplots = 4 ax_rollout = fig.add_subplot(1, n_subplots, i_subplot) i_subplot += 1 h = plotRollout(rollout_eval.cost_vars, ax_rollout) setColor(h, i_update, n_updates) ax_space = fig.add_subplot(1, n_subplots, i_subplot) i_subplot += 1 highlight = (i_update == 0) plotUpdate(distribution, cost_eval, samples, costs, weights, distribution_new, ax_space, highlight) plotExplorationCurve(exploration_curve, fig.add_subplot(1, n_subplots, i_subplot)) plotLearningCurve(learning_curve, fig.add_subplot(1, n_subplots, i_subplot + 1))
return weights if __name__ == "__main__": eliteness = 10 weighting_method = 'PI-BB' covar_decay_factor = 0.8 updater = UpdaterCovarDecay(eliteness, weighting_method, covar_decay_factor) diagonal_min = 0.1 diagonal_max = 1.0 diag_only = False learning_rate = 1.0 updater = UpdaterCovarAdaptation(eliteness, weighting_method, diagonal_max, diagonal_min, diag_only, learning_rate) mu = np.array([2, 4]) cov = np.array([[0.3, 0.0], [0.0, 0.5]]) distribution = DistributionGaussian(mu, cov) n_samples = 10 samples = distribution.generateSamples(n_samples) costs = abs(samples[:, 0]) + abs(samples[:, 1]) # Manhattan distance (new_distribution, w) = updater.updateDistribution(distribution, samples, costs) print(distribution.covar) print(new_distribution.covar)
if (len(sys.argv)>3): sigma = float(sys.argv[4]) n_samples = 10 if (len(sys.argv)>4): n_samples = int(sys.argv[5]) print("Python | calling "+" ".join(sys.argv)) print('Python | Loading mean from "'+input_parameters_file+'"') parameter_vector = np.loadtxt(input_parameters_file) print('Python | Generating '+str(n_samples)+' samples with sigma='+str(sigma)) covar_init = sigma*sigma*np.eye(parameter_vector.size) distribution = DistributionGaussian(parameter_vector, covar_init) samples = distribution.generateSamples(n_samples) print('Python | Saving covar to "'+output_covar_file+'"') np.savetxt(output_covar_file,covar_init) print('Python | Saving samples to '+output_directory) for i_sample in range(n_samples): rollout_directory = '%s/rollout%03d/' % (output_directory, i_sample+1) sample_filename = rollout_directory+'policy_parameters.txt' print('Python | Saving sample '+str(i_sample)+' to '+sample_filename) if not os.path.exists(rollout_directory): os.makedirs(rollout_directory) np.savetxt(sample_filename,samples[i_sample,:])
def plotOptimizationRollouts(directory, fig, plotRollout=None, plot_all_rollouts=False, dimensionality_reduction='SVD'): if not fig: fig = plt.figure(1, figsize=(9, 4)) # Determine number of updates n_updates = 1 update_dir = '%s/update%05d' % (directory, n_updates) while containsNewDistribution(update_dir): n_updates += 1 update_dir = '%s/update%05d' % (directory, n_updates) n_updates -= 1 if n_updates < 2: return None all_costs = [] learning_curve = [] exploration_curve = [] i_samples = 0 if dimensionality_reduction != None: # Do dimensionality reduction by finding the 2 principal direction out of the n-dimensional set of data (means of gaussian distributions) # 1) Fetch the data from folder and them stack them in the array time_series_distribution # 2) Do dimensionality reduction using SVD or TSNE # 1) time_series_distribution = [] for i_update in range(n_updates + 1): update_dir = '%s/update%05d' % (directory, i_update) # Read data try: mean = np.loadtxt(update_dir + "/distribution_mean.txt") time_series_distribution.append(mean) except IOError: print("IOError") return (-1) # 2) if dimensionality_reduction == 'SVD': # time_series_distribution = PCA(n_components=2).fit_transform(time_series_distribution) U, S, V = np.linalg.svd(time_series_distribution, full_matrices=True) # Truncated SVD time_series_distribution = np.matmul(U[:, 0:2], np.diag(S[0:2])) print("manual truncation ", time_series_distribution) elif dimensionality_reduction == 'TSNE': time_series_distribution = TSNE( n_components=2, perplexity=5).fit_transform(time_series_distribution) for i_update in range(n_updates): update_dir = '%s/update%05d' % (directory, i_update) if dimensionality_reduction != None: try: # Read data mean = time_series_distribution[i_update, :] covar = np.loadtxt(update_dir + "/distribution_covar.txt") except IOError: distribution_new = None if dimensionality_reduction == 'SVD': # change coordinates for the covariance matrix into principal directions using V covar = np.matmul(np.matmul(np.transpose(V), covar), V) distribution = DistributionGaussian(mean, covar[0:2, 0:2]) elif dimensionality_reduction == 'TSNE': # for tSNE we have no choice, do an SVD of the covariance matrix and take its principal singular values U_tSNE, S_tSNE, V_tSNE = np.linalg.svd(covar, full_matrices=True) distribution = DistributionGaussian(mean, np.diag(S_tSNE[0:2])) try: mean = time_series_distribution[i_update + 1, :] covar = np.loadtxt(update_dir + "/distribution_new_covar.txt") except IOError: distribution_new = None if dimensionality_reduction == 'SVD': # change coordinates for the covariance matrix into principal directions using V covar = np.matmul(np.matmul(np.transpose(V), covar), V) distribution_new = DistributionGaussian(mean, covar[0:2, 0:2]) elif dimensionality_reduction == 'TSNE': # for tSNE we have no choice, do an SVD of the covariance matrix and take its principal singular values U_tSNE, S_tSNE, V_tSNE = np.linalg.svd(covar, full_matrices=True) distribution_new = DistributionGaussian( mean, np.diag(S_tSNE[0:2])) else: # Read data mean = np.loadtxt(update_dir + "/distribution_mean.txt") covar = np.loadtxt(update_dir + "/distribution_covar.txt") distribution = DistributionGaussian(mean, covar) try: mean = np.loadtxt(update_dir + "/distribution_new_mean.txt") covar = np.loadtxt(update_dir + "/distribution_new_covar.txt") distribution_new = DistributionGaussian(mean, covar) except IOError: distribution_new = None try: covar_block_sizes = np.loadtxt(update_dir + "/covar_block_sizes.txt") except IOError: covar_block_sizes = len(distribution.mean) try: samples = np.loadtxt(update_dir + "/samples.txt") except IOError: samples = None costs = np.loadtxt(update_dir + "/costs.txt") weights = np.loadtxt(update_dir + "/weights.txt") try: cost_eval = np.loadtxt(update_dir + "/cost_eval.txt") except IOError: cost_eval = None n_rollouts = len(weights) rollouts = [] for i_rollout in range(n_rollouts): rollout_dir = '%s/rollout%03d' % (update_dir, i_rollout + 1) rollouts.append(loadRolloutFromDirectory(rollout_dir)) rollout_eval = loadRolloutFromDirectory(update_dir + '/rollout_eval/') # Update learning curve # Bookkeeping and plotting # All the costs so far all_costs.extend(costs) # Update exploration curve i_samples = i_samples + n_rollouts cur_exploration = np.sqrt(distribution.maxEigenValue()) exploration_curve.append([i_samples, cur_exploration]) # Update learning curve learning_curve.append([i_samples]) learning_curve[-1].extend(np.atleast_1d(cost_eval)) n_subplots = 3 i_subplot = 1 if plotRollout: n_subplots = 4 ax_rollout = fig.add_subplot(1, n_subplots, i_subplot) i_subplot += 1 h = plotRollout(rollout_eval.cost_vars, ax_rollout) setColor(h, i_update, n_updates) ax_space = fig.add_subplot(1, n_subplots, i_subplot) i_subplot += 1 highlight = (i_update == 0) plotUpdate(distribution, cost_eval, samples, costs, weights, distribution_new, ax_space, highlight, plot_samples=False) plotExplorationCurve(exploration_curve, fig.add_subplot(1, n_subplots, i_subplot)) plotLearningCurve(learning_curve, fig.add_subplot(1, n_subplots, i_subplot + 1))
def plotOptimizationRollouts(directory,fig,plotRollout=None,plot_all_rollouts=False): if not fig: fig = plt.figure(1,figsize=(9, 4)) # Determine number of updates n_updates = 1 update_dir = '%s/update%05d' % (directory, n_updates) while containsNewDistribution(update_dir): n_updates += 1 update_dir = '%s/update%05d' % (directory, n_updates) n_updates -= 1 if n_updates<2: return None all_costs = [] learning_curve = [] exploration_curve = [] i_samples = 0 for i_update in range(n_updates): update_dir = '%s/update%05d' % (directory, i_update) # Read data mean = np.loadtxt(update_dir+"/distribution_mean.txt") covar = np.loadtxt(update_dir+"/distribution_covar.txt") distribution = DistributionGaussian(mean,covar) try: mean = np.loadtxt(update_dir+"/distribution_new_mean.txt") covar = np.loadtxt(update_dir+"/distribution_new_covar.txt") distribution_new = DistributionGaussian(mean,covar) except IOError: distribution_new = None try: covar_block_sizes = np.loadtxt(update_dir+"/covar_block_sizes.txt") except IOError: covar_block_sizes = len(distribution.mean) try: samples = np.loadtxt(update_dir+"/samples.txt") except IOError: samples = None costs = np.loadtxt(update_dir+"/costs.txt") weights = np.loadtxt(update_dir+"/weights.txt") try: cost_eval = np.loadtxt(update_dir+"/cost_eval.txt") except IOError: cost_eval = None n_rollouts = len(weights) rollouts = [] for i_rollout in range(n_rollouts): rollout_dir = '%s/rollout%03d' % (update_dir, i_rollout+1) rollouts.append(loadRolloutFromDirectory(rollout_dir)) rollout_eval = loadRolloutFromDirectory(update_dir+'/rollout_eval/') # Update learning curve # Bookkeeping and plotting # All the costs so far all_costs.extend(costs) # Update exploration curve i_samples = i_samples + n_rollouts cur_exploration = np.sqrt(distribution.maxEigenValue()) exploration_curve.append([i_samples,cur_exploration]) # Update learning curve learning_curve.append([i_samples]) learning_curve[-1].extend(np.atleast_1d(cost_eval)) n_subplots = 3 i_subplot = 1 if plotRollout: n_subplots = 4 ax_rollout = fig.add_subplot(1,n_subplots,i_subplot) i_subplot += 1 h = plotRollout(rollout_eval.cost_vars,ax_rollout) setColor(h,i_update,n_updates) ax_space = fig.add_subplot(1,n_subplots,i_subplot) i_subplot += 1 highlight = (i_update==0) plotUpdate(distribution,cost_eval,samples,costs,weights,distribution_new,ax_space,highlight) plotExplorationCurve(exploration_curve,fig.add_subplot(1,n_subplots,i_subplot)) plotLearningCurve(learning_curve,fig.add_subplot(1,n_subplots,i_subplot+1))