class Visualizer: def __init__(self, working_dir, config_space_file): self.working_dir = working_dir self.config_space_file = config_space_file self.result_logger = ResultLogger(working_dir=working_dir) self.config_space = ExperimentArguments.read_configuration_space(config_space_file) self.results = {} experiment_start_time = np.inf self.alpha = 0.4 self.size = 40 colors = ["windows blue", "amber", "green", "red", ] self.pal = sns.xkcd_palette(colors) self.lines = {} # First find starting time for configuration, result in self.result_logger.get_results(self.config_space): budget, result_timestamps, result, exception = result t_started = int(result_timestamps['started']) experiment_start_time = min(t_started, experiment_start_time) for configuration, result, info in self.result_logger.get_results(self.config_space, with_info=True): budget, result_timestamps, result, exception = result if result is None: result = np.inf else: result = 100 + float(result['loss']) * 100 #result = np.sqrt(float(result['loss']) * stdv**2) t_finished = int(result_timestamps['finished']) point = ((t_finished - experiment_start_time) / 3600, result, info['model_based_pick']) if point[0] > 170: continue try: self.results[budget].append(point) except KeyError: self.results[budget] = [point] try: self.lines[configuration].append(point) except KeyError: self.lines[configuration] = [point] self.best_points = {} for key, values in self.results.items(): self.best_points[key] = [] values = sorted(values, key=lambda c: c[0]) best = np.inf for time, result, mb in values: if result < best: best = result self.best_points[key].append((time, result)) def plot(self): for c, (key, values) in zip(self.pal, sorted(self.results.items())): print(c) mb_values = [v[:2] for v in values if v[2] is True] mf_values = [v[:2] for v in values if v[2] is False] try: print('A') x, y = [list(a) for a in zip(*mb_values)] plt.scatter(x, y, c=c, s=self.size, linewidths=2, edgecolors='black', alpha=self.alpha, label=str('Model Based %s' % int(key))) #sns.regplot(np.array(x), np.array(y), color=c, ci=99) except Exception as e: print(e) pass try: x, y = [list(a) for a in zip(*mf_values)] plt.scatter(x, y, c=c, s=self.size, alpha=self.alpha, label=str('Model Free %s' % int(key))) except Exception as e: print(e) pass plt.xlim(-1, 80) plt.ylim(-1, 80) plt.xlabel("Time [hours]", fontsize=23) plt.ylabel("Classification Error [%]", fontsize=23) # #plt.ylabel("RMSE", fontsize=25) plt.tick_params(axis='both', which='major', labelsize=22) plt.tick_params(axis='both', which='minor', labelsize=21) def plot_best_points(self): for c, (key, values) in zip(self.pal, sorted(self.best_points.items())): x, y = [list(a) for a in zip(*values)] plt.step(x, y, c=c, linewidth=3, where='post', alpha=0.8) print(y) def plot_lines(self): for key, value in self.lines.items(): if len(value) > 1: x = [v[0] for v in value] y = [v[1] for v in value] plt.plot(x, y, c='gray', alpha=0.2)
class Visualizer: def __init__(self, working_dir, config_space_file): self.working_dir = working_dir self.config_space_file = config_space_file self.result_logger = ResultLogger(working_dir=working_dir) self.config_space = ExperimentArguments.read_configuration_space( config_space_file) self.results = {} experiment_start_time = np.inf self.alpha = 0.8 self.size = 50 # First find starting time for configuration, result in self.result_logger.get_results( self.config_space): budget, result_timestamps, result, exception = result t_started = int(result_timestamps['started']) experiment_start_time = min(t_started, experiment_start_time) for configuration, result, info in self.result_logger.get_results( self.config_space, with_info=True): budget, result_timestamps, result, exception = result if result is None: result = np.inf else: result = float(result['loss']) t_finished = int(result_timestamps['finished']) point = ((t_finished - experiment_start_time) / 3600, result, info['model_based_pick']) try: self.results[budget].append(point) except KeyError: self.results[budget] = [point] self.best_points = {} for key, values in self.results.items(): self.best_points[key] = [] values = sorted(values, key=lambda c: c[0]) best = np.inf for time, result, mb in values: if result < best: best = result self.best_points[key].append((time, result)) def plot(self): pal = sns.color_palette(n_colors=len(self.results.keys())) for c, (key, values) in zip(pal, sorted(self.results.items())): mb_values = [v[:2] for v in values if v[2] is True] mf_values = [v[:2] for v in values if v[2] is False] x, y = [list(a) for a in zip(*mb_values)] plt.scatter(x, y, c=c, s=self.size, linewidths=2, edgecolors='black', alpha=self.alpha, label=str('Model Based %s' % key)) x, y = [list(a) for a in zip(*mf_values)] plt.scatter(x, y, c=c, s=self.size, alpha=self.alpha, label=str('Model Free %s' % key)) def plot_best_points(self): pal = sns.color_palette(n_colors=len(self.best_points.keys())) for c, (key, values) in zip(pal, sorted(self.best_points.items())): x, y = [list(a) for a in zip(*values)] plt.step(x, y, c=c, linewidth=3)