def plot(self): assert (self.input_space.num_dims == 2 or self.input_space.num_dims == 1) size = 50 x = np.linspace(0, 1, size) y = np.linspace(0, 1, size) X, Y = np.meshgrid(x, y) if self.input_space.num_dims == 2: k = 0 for key in self.models: Z = np.zeros((size, size)) for i in range(size): for j in range(size): params = self.input_space.from_unit( np.array([X[i, j], Y[i, j]])).flatten() Z[i, j] = self.f( paramify_no_types( self.input_space.paramify(params)))[key] plt.figure() im = plt.imshow(Z, interpolation='bilinear', origin='lower', cmap=cm.gray, extent=(0, 1, 0, 1)) CS = plt.contour(X, Y, Z) plt.clabel(CS, inline=1, fontsize=10) plt.title(str(key)) plt.show() k += 1 else: k = 0 for key in self.models: Z = np.zeros(size) for i in range(size): params = self.input_space.from_unit(np.array( [x[i]])).flatten() Z[i] = self.f( paramify_no_types( self.input_space.paramify(params)))[key] plt.figure() plt.plot(x, Z, color='red', marker='.', markersize=1) plt.title(str(key)) plt.show() k += 1
def python_launcher(job): # Run a Python function sys.stderr.write("Running python job.\n") # Add directory to the system path. sys.path.append(os.path.realpath(job['expt_dir'])) # Change into the directory. os.chdir(job['expt_dir']) sys.stderr.write("Changed into dir %s\n" % (os.getcwd())) # Strip off the data types and just get the raw values params = paramify_no_types(job['params']) # Load up this module and run main_file = job['main-file'] if main_file[-3:] == '.py': main_file = main_file[:-3] sys.stderr.write('Importing %s.py\n' % main_file) module = __import__(main_file) sys.stderr.write('Running %s.main()\n' % main_file) result = module.main(job['id'], params) # Change back out. os.chdir('..') sys.stderr.write("Got result %s\n" % (result)) return result
def fun(params, gradient = False): if len(params.shape) > 1 and params.shape[ 1 ] > 1: params = params.flatten() params = input_space.from_unit(np.array([ params ])).flatten() return -1.0 * module.main(0, paramify_no_types(input_space.paramify(params)))[ task ]
def fun(params, gradient = False): if len(params.shape) > 1 and params.shape[ 1 ] > 1: values = np.zeros(params.shape[ 0 ]) params_orig = params for i in range(params_orig.shape[ 0 ]): param = params[ i, : ] param = param.flatten() param = input_space.from_unit(np.array([ param ])).flatten() values[ i ] = module.main(0, paramify_no_types(input_space.paramify(param)))[ task ] else: return module.main(0, paramify_no_types(input_space.paramify(params)))[ task ] return values
def true_func(stored, module, violation_value=np.inf, constraint_tol=0.0, objective=DEFAULTS["task_name"], constraints=[]): params = stored['params'] # if we don't actually know the true objective, just return what we think is the best if not hasattr(module, 'true_func'): if stored['objective'] is None: return np.nan else: return stored['objective'] # output = module.main(0, paramify_no_types(params)) # otherwise, assume we know it output = module.true_func(0, paramify_no_types(params)) if not isinstance(output, dict): if len(constraints) != 0: raise Exception( "output is not a dict and yet you said there were constraints..?" ) val = output else: val = output[objective] for c in constraints: if output[c] < -constraint_tol: print 'Violation value of %f for constraint %s' % (output[c], c) # print str(paramify_no_types(params)) return violation_value # if the module defines the true solution value, then use the gap. otherwise don't return np.abs( val - module.true_val()) # abs not really needed, it will always be positive
def main(dirs, n_repeat=-1, n_iter_spec=None, rec_type="model", average="mean", log_scale=False, violation_value=1., constraint_tol=0., make_dist_plot=False, mainfile=None, stretch_x=False, task_comp_x=None, plot_wall_time=False, bin_size=1.0, plot_separate=False, labels=None, y_axis_label=None, x_axis_label=None): # Create the figure that plots utility gap fig = dict() ax = dict() # averaging function if average == "mean": avg = np.mean elif average == "median": avg = np.median else: raise Exception("Unknown average %s" % average) fig['err'] = plt.figure() ax['err'] = fig['err'].add_subplot(1, 1, 1) if plot_wall_time: ax['err'].set_xlabel("wall time (min)", size=25) elif x_axis_label: ax['err'].set_xlabel(x_axis_label, size=25) else: ax['err'].set_xlabel('Number of function evaluations', size=25) ax['err'].tick_params(axis='both', which='major', labelsize=20) # Create the figure that plots L2 distance from solution fig['dist'] = plt.figure() ax['dist'] = fig['dist'].add_subplot(1, 1, 1) if x_axis_label: ax['dist'].set_xlabel(x_axis_label, size=25) else: ax['dist'].set_xlabel('Number of function evaluations', size=25) if y_axis_label: ax['dist'].set_ylabel(y_axis_label, size=25) elif log_scale: ax['dist'].set_ylabel('$\log_{10}\, \ell_2$-distance', size=25) else: ax['dist'].set_ylabel('$\ell_2$-distance', size=25) ax['dist'].tick_params(axis='both', which='major', labelsize=20) db_document_name = 'recommendations' acq_names = list() for expt_dir in dirs: options = parse_config_file(expt_dir, 'config.json') experiment_name = options["experiment_name"] input_space = InputSpace(options["variables"]) chooser_module = importlib.import_module('spearmint.choosers.' + options['chooser']) chooser = chooser_module.init(input_space, options) db = MongoDB(database_address=options['database']['address']) jobs = load_jobs(db, experiment_name) hypers = db.load(experiment_name, 'hypers') tasks = parse_tasks_from_jobs(jobs, experiment_name, options, input_space) if rec_type == "model": if mainfile is None: main_file = options['main_file'] else: main_file = mainfile sys.path.append(options['main_file_path'] ) # TODO: make this nicer with proper importin if (main_file[-3:] == u'.py') is True: module = importlib.import_module(main_file[:len(main_file) - 3]) else: module = importlib.import_module(main_file) sys.path.remove(options['main_file_path']) obj, con = get_objectives_and_constraints(options) # get the names obj = obj[0] # only one objective print 'Found %d constraints' % len(con) plot_utility_gap = rec_type == "model" and hasattr(module, 'true_val') if plot_utility_gap: print 'PLOTTING UTILITY GAP' if y_axis_label: ax['err'].set_ylabel(y_axis_label, size=25) elif log_scale: ax['err'].set_ylabel('$\log_{10}$ utility gap', size=25) else: ax['err'].set_ylabel('utility gap', size=25) else: if y_axis_label: ax['err'].set_ylabel(y_axis_label, size=25) elif log_scale: ax['err'].set_ylabel('$\log_{10}$ objective value', size=25) else: ax['err'].set_ylabel('objective value', size=25) # Make the directory for the plots plots_dir = os.path.join(expt_dir, 'plots') if not os.path.isdir(plots_dir): os.mkdir(plots_dir) # if the module provides the location of the true solution, plot the distance to this solution vs iterations if make_dist_plot and not hasattr(module, 'true_sol'): raise Exception( "make_dist_plot turned on but cannot find true sol in the main_file" ) # If repeat >= 0, then we are averaging a number of experiments # We assume the experiments are stored with the original name plus a hyphen plus the number n_repeat = int(n_repeat) if n_repeat < 0: recs = db.load(experiment_name, db_document_name) if recs is None: raise Exception( "Could not find experiment %s in database at %s" % (experiment_name, options['database']['address'])) # the x axis represents the number of evals of a particular task given by task_comp_x # so we only take the data where this number was incrememted, i.e. when this task was evaluated if task_comp_x: # only include recommendations when you finish a particular task new_recs = list() last_complete = 0 for rec in recs: cur_complete = rec['num_complete_tasks'][task_comp_x] if cur_complete > last_complete: last_complete = cur_complete new_recs.append(rec) recs = new_recs n_iter = len(recs) if n_iter_spec is None else n_iter_spec iters = range(n_iter) if plot_wall_time: if task_comp_x: raise Exception("Do not use plot wall_time with task_x") iters = [rec['total_elapsed_time'] / 60.0 for rec in recs] iters = iters[:n_iter] iters = np.array(iters, dtype=float) print 'Found %d iterations' % len(recs) if rec_type == "model": values = [ true_func(rec, module, violation_value, constraint_tol, obj, con) for rec in recs ] if log_scale: ax['err'].plot(iters, map(np.log10, values)) else: ax['err'].plot(iters, values) else: if rec_type == "observations": observations = [x['obj_o'] for x in recs] elif rec_type == "mixed": observations = [x['obj_om'] for x in recs] else: raise Exception("unknown rec type") for i in xrange(len(observations)): if observations[i] is None or np.isnan(observations[i]): observations[i] = violation_value # print observations # print len(observations) if log_scale: ax['err'].plot(iters, np.log10(observations)) else: ax['err'].plot(iters, observations) if make_dist_plot: distances = [ params_norm(rec['params'], module.true_sol()) for rec in recs ] if log_scale: ax['dist'].plot(iters, np.log10(distances)) else: ax['dist'].plot(iters, distances) else: # MULTIPLE REPEATS repeat_recs = [ db.load(repeat_experiment_name(experiment_name, j), db_document_name) for j in xrange(n_repeat) ] if None in repeat_recs: for i, repeat_rec in enumerate(repeat_recs): if repeat_rec is None: print 'Could not load experiment %s repeat %d' % ( experiment_name, i) print 'Exiting...' return if task_comp_x: # only include recommendations when you finish a particular task new_repeat_recs = list() for recs in repeat_recs: recs = sorted(recs, key=lambda k: k['id']) # sort by id new_recs = list() last_complete = 0 for rec in recs: cur_complete = rec['num_complete_tasks'][task_comp_x] if cur_complete == last_complete + 1: last_complete = cur_complete new_recs.append(rec) elif cur_complete == last_complete: pass else: print( 'WARNING: cur complete=%d, last_complete=%d' % (cur_complete, last_complete)) break new_repeat_recs.append(new_recs) repeat_recs = new_repeat_recs n_iter_each = map(len, repeat_recs) if plot_wall_time: """ do everything separately from here if plotting wall time here is what we do... we can't have a square array because we don't want to take the minimum number of iterations... we want to take ALL iterations for each repeat, and this number may be different for different repeats. so we store all times/values in a list of arrays then we chop things up into bins """ if rec_type != "model": values = list() wall_times = list() for j in xrange( n_repeat): # loop over repeated experiments wall_times.append( np.array([ repeat_recs[j][i]['total_elapsed_time'] / 60.0 for i in xrange(n_iter_each[j]) ])) if rec_type == "observations": values.append([ repeat_recs[j][i]['obj_o'] for i in xrange(n_iter_each[j]) ]) elif rec_type == "mixed": values.append([ repeat_recs[j][i]['obj_om'] for i in xrange(n_iter_each[j]) ]) else: raise Exception("unknown rec type") for i in xrange(n_iter_each[j]): if values[-1][i] is None or np.isnan( values[-1][i]): values[-1][i] = violation_value values[-1] = np.array(values[-1]) # print values else: # if plot wall tiem but using model values = list() wall_times = list() for j in xrange( n_repeat): # loop over repeated experiments # for this repeat, get all wall times wall_times.append( np.array([ repeat_recs[j][i]['total_elapsed_time'] / 60.0 for i in xrange(n_iter_each[j]) ])) values_j = np.zeros(n_iter_each[j]) for i in xrange( n_iter_each[j]): # loop over iterations val = true_func(repeat_recs[j][i], module, None, constraint_tol, obj, con) if val is None or np.isnan( val ): #set to violation value here so we can print out this info... values_j[i] = violation_value print 'Violation with params %s at repeat %d iter %d' % ( paramify_no_types( repeat_recs[j][i]['params']), j, i) else: values_j[i] = val values.append(values_j) # change the data structure to be time bins and include everything in # those time bins across repeats end_times = map(max, wall_times) for j in xrange(n_repeat): print 'end time for repeat %d: %f' % (j, end_times[j]) iters = np.arange(0.0, np.round(max(end_times)), bin_size) new_values = list() for i, timestep in enumerate(iters): # print 'Creating wall time bin from %f to %f. (%d/%d)' % (i, i+bin_size, i, len(iters)) new_value = list() for j in xrange(n_repeat): new_value = np.append( new_value, values[j][np.logical_and( wall_times[j] >= timestep, wall_times[j] < timestep + bin_size)].flatten()) # if a time bin is empty across all repeats: if len(new_value) == 0: if i == 0: new_value = [violation_value] else: new_value = new_values[-1] new_values.append(new_value) values = new_values # make the first value equal to the violation value (optional) iters = np.append(iters, max(iters) + bin_size) values.insert(0, np.array([violation_value])) # Average over the repeated experiments average_values = map(avg, values) errorbars = bootstrap_errorbars(values, log=log_scale, avg=avg) # plt.yscale('log', nonposy='clip') if log_scale: ax['err'].errorbar(iters, np.log10(average_values), yerr=errorbars) else: ax['err'].errorbar(iters, average_values, yerr=errorbars) else: # NOT WALL TIME n_iter = reduce(min, n_iter_each, np.inf) if n_iter_spec is None: print 'Found %d repeats with at least %d iterations' % ( n_repeat, n_iter) print {i: n_iter_each[i] for i in xrange(n_repeat)} elif n_iter < n_iter_spec: print 'You specified %d iterations but there are only %d available... so plotting %d' % ( n_iter_spec, n_iter, n_iter) else: n_iter = n_iter_spec print 'Plotting %d iterations' % n_iter iters = range(n_iter) if rec_type != "model": values = np.zeros((n_iter, n_repeat)) for j in xrange( n_repeat): # loop over repeated experiments for i in iters[j]: # loop over iterations if rec_type == "observations": values[i, j] = repeat_recs[j][i]['obj_o'] elif rec_type == "mixed": values[i, j] = repeat_recs[j][i]['obj_om'] else: raise Exception("unknown rec type") if values[i, j] is None or np.isnan(values[i, j]): values[i, j] = violation_value print values else: values = np.zeros((n_iter, n_repeat)) distances = np.zeros((n_iter, n_repeat)) for j in xrange( n_repeat): # loop over repeated experiments for i in iters: # loop over iterations val = true_func(repeat_recs[j][i], module, None, constraint_tol, obj, con) if val is None: #set to violation value here so we can print out this info... values[i, j] = violation_value print 'Violation with params %s at repeat %d iter %d' % ( paramify_no_types( repeat_recs[j][i]['params']), j, i) else: values[i, j] = val if make_dist_plot: distances[i, j] = params_norm( repeat_recs[j][i]['params'], module.true_sol()) if plot_separate: if log_scale: ax['err'].plot(iters, np.log10(values)) else: ax['err'].plot(iters, values) else: # Average over the repeated experiments average_values = map(avg, values) errorbars = bootstrap_errorbars(values, log=log_scale, avg=avg) # plt.yscale('log', nonposy='clip') if stretch_x: fctr = float(n_iter_spec) / float(n_iter) iters = np.array(iters) * fctr print 'Stretching x axis by a factor of %f' % fctr if log_scale: ax['err'].errorbar(iters, np.log10(average_values), yerr=errorbars) else: ax['err'].errorbar(iters, average_values, yerr=errorbars) if make_dist_plot: average_dist = map(avg, distances) errorbars_dist = bootstrap_errorbars(distances, log=log_scale, avg=avg) if log_scale: ax['dist'].errorbar(iters, np.log10(average_dist), yerr=errorbars_dist) else: ax['dist'].errorbar(iters, average_dist, yerr=errorbars_dist) acq_names.append(options["tasks"].values()[0]["acquisition"]) if acq_names[-1] == 'PES': acq_names[-1] = 'PESC' if acq_names[-1] == 'ExpectedImprovement': acq_names[-1] = 'EIC' if labels: ax['err'].legend(labels.split(';'), fontsize=16, loc='lower left') ax['dist'].legend(labels.split(';'), fontsize=20) elif len(acq_names) > 1: ax['err'].legend(acq_names, fontsize=20) ax['dist'].legend(acq_names, fontsize=20) # save it in the last directory... (if there are multiple directories) if not plot_wall_time: if n_repeat >= 0: print 'Made a plot with %d repeats and %d iterations' % (n_repeat, n_iter) else: print 'Made a plot with %d iterations' % (n_iter) else: if n_repeat >= 0: print 'Made a plot with %d repeats and %f minutes' % (n_repeat, max(iters)) else: print 'Made a plot with %f minutes' % (max(iters)) file_prefix = '%s_' % average if n_repeat > 0 else '' file_postfix = '_wall_time' if plot_wall_time else '' fig['err'].tight_layout() figname = os.path.join(plots_dir, '%serror%s' % (file_prefix, file_postfix)) fig['err'].savefig(figname + '.pdf') fig['err'].savefig(figname + '.svg') print 'Saved to %s' % figname if make_dist_plot: fig['dist'].tight_layout() figname_dist = os.path.join( plots_dir, '%sl2_distance%s.pdf' % (file_prefix, file_postfix)) fig['dist'].savefig(figname_dist) print 'Saved to %s' % figname_dist