def main(): fs = FileSystem() data = fs.read_json('outputs', 'benchmark.json') if False: view_factors(data, ['seed', 'bs', 'T', 'lr', 'mtm', 'wd']) # Varying factors are [seed, bs, T, lr] params = {'m': 10, 'n': 5, 'p': 12} ax_h_var = 'lr' ax_v_var = 'bs' line_var = 'T' fig_var = 'seed' x_var = 'loss' legend_tags = ['T'] s2v_loss_grid(data, x_var, ax_h_var, ax_v_var, fig_var, line_var, legend_tags, params)
def plot_benchmark(): loc = 'output' tag = 'benchmark_plnn_res' tag = 'benchmark_mnist_res' fs = FileSystem() fpath = fs._infer_location(loc) fnames = [x for x in os.listdir(fpath) if x.startswith(tag)] results = [] for fname in fnames: res = fs.read_json(loc, fname) results.append(res) vis = visdom.Visdom() plot_in_visdom(vis, results)
def __init__(self, topology, service, environment, alternative): Thread.__init__(self) """ Utils objects """ # Get the object for filesystem handling self._fs = FileSystem.get_instance() # Logger self._log = Logger.get_instance() # Root simulation path (simulation/) self._root_simulation_path = self._fs.get_simulations_folder() # Specific simulation path (simulation/service_name/timestamp/) self._simulation_path = None # The topology self._topology = topology # Service to evaluate self._service = service # The environment in which this simulation is running self._environment = environment # The alternative of the service to evaluate self._alternative = alternative # The metrics to evaluate during this simulation self._metrics = alternative.get_metrics() # Extractor count. This variable is used to keep track of how many extractors notified this object self._extractor_number = len(self._metrics) self._extractor_count = 0 # Initialize the simulation self._init()
def __init__(self, controller_path, controller_cmd): # Get the framework file system handler self._fs = FileSystem.get_instance() # Logger self._log = Logger.get_instance() # Controller's parameters self._path = controller_path self._cmd = controller_cmd # The controller process self._controller_process = None
def get_mnist_info(): # Get paths fs = FileSystem() fname = 'mnist_info.json' if fs.exists('mnist', fname): info = fs.read_json('mnist', fname) else: print('MNIST info does not exist') fpaths, fdirs = DatasetPLNN.get_mps_paths(ext='.info', dataset='mnist') print(len(fpaths)) info = [] for fpath in tqdm(fpaths): d = infopath_2_params(fpath) info.append(d) fs.write_json(info, 'mnist', 'mnist_info.json') return info
def __init__(self): Extractor.__init__(self) # The FileSystem handler self._fs = FileSystem.get_instance()
def s2v_loss_grid(data, x_var, ax_row_var, ax_col_var, fig_var, line_var, legend_tags, fixed_params): ''' fig_data is a list of dictionaries containing information about the figure. fig_data is a list of dictionaries Function creates a sequence of subplots with several plots in them fig_vars[0] ... fig_vars[k] ========================= ========================= | ax_data[0] | ax_data[1] | | ax_data[0] | ax_data[1] | | | | | | | | | | | | | ------------------------- ... ------------------------- | ... | ax_data[n] | | ... | ax_data[n] | | | | | | | | | | | | | ========================= ========================= ''' fs = FileSystem() m = fixed_params['m'] n = fixed_params['n'] p = fixed_params['p'] # title lp_title = lambda m, n, p : '(m, n, p) = (%d, %d, %d)' % (m, n, p) opt_title = lambda a, b, c : '(lr, mtm, wd) = (%g, %g, %g)' % (a, b, c) comments = 'T: number of s2v rounds, acc: test-accuracy' # formatter def formatter(b): if isinstance(b, int): return '%d' % (b) elif isinstance(b, float): return '%g' % (b) elif isinstance(b, str): return '%s' % (b) else: raise ValueError # --------------- # Parse fig_data by levels # 1. Figure # 2. Axis (horizontal, vertical) # 3. Line # --------------- ds = {} # split figures for d in data: key = d[fig_var] if key in ds: ds[key].append(d) else: ds[key] = [d] # Line variables line_vars = list(set([x[line_var] for x in data])) # subplots for fig_val in ds.keys(): figdata = ds[fig_val] ax_rows = sorted(list(set([x[ax_row_var] for x in figdata]))) ax_cols = sorted(list(set([x[ax_col_var] for x in figdata]))) f, axs = plt.subplots(len(ax_rows), len(ax_cols), sharex='col', sharey='row') for i, j in itertools.product(range(len(ax_rows)), range(len(ax_cols))): ax_r = ax_rows[i] ax_c = ax_cols[j] plot_data = [d for d in figdata if d[ax_row_var] == ax_r and d[ax_col_var] == ax_c] legends = [] for t, d in enumerate(plot_data): x = eval(d[x_var]) lines = axs[i, j].plot(range(len(x)), x, alpha=0.15*(t+1), label='t=%d' % (t)) # yscale axs[i, j].set_yscale('log') # Rotate ticks plt.setp(axs[i, j].get_xticklabels(), rotation=30, horizontalalignment='right') # legends legends.append(', '.join(['%s:%s' % (var, formatter(d[var])) for var in legend_tags])) # inline label idx = math.ceil(len(x)/(2*len(plot_data) + 1) * (2*t + 1)) color = lines[len(lines)-1].get_color() axs[i, j].text(idx, x[idx], 'acc=%s' % (formatter(d['acc'])), color = color, alpha=1) # get legends if i == len(ax_rows) - 1 and j == len(ax_cols) - 1: axs[i, j].legend(legends, loc='lower left') # cosmetics axs[i, j].set_xlabel('Iterations') axs[i, j].set_ylabel('log(loss(yhat, y))') # Label axes col_tags = ['%s=%s' % (ax_col_var, formatter(x)) for x in ax_cols] row_tags = ['%s=%s' % (ax_row_var, formatter(x)) for x in ax_rows] for ax, col in zip(axs[0], col_tags): ax.set_title(col) for ax, row in zip(axs[:,0], row_tags): ax.set_ylabel(row, rotation=90, size='large') plt.suptitle(lp_title(m, n, p) + " / " + comments, fontsize=15) plt.tight_layout(rect=[0, 0.03, 1, 0.95]) fpath = fs.get_path('outputs', 'overfitting_s2v_benchmark_%s_%s.png' % (fig_var, str(fig_val))) plt.savefig(fpath) return
def __init__(self): # FileSystem handler self._fs = FileSystem.get_instance() # Logger self._log = Logger.get_instance()
def __init__(self): Collector.__init__(self) self._sniffer = None # Logger self._fs = FileSystem.get_instance()
def __init__(self): # The reference to the FileSystem object, useful for writing configuration in tmp folder self._fs = FileSystem.get_instance() # Logger self._log = Logger.get_instance()