class VisdomLinePlotter(object): """Plots to Visdom""" def __init__(self, env_name='main', plot_path='/tmp'): self.viz = Visdom(port=6065) self.env = env_name self.plots = {} self.plot_path = plot_path #self.viz.log_to_filename = log_filename def plot(self, var_name, split_name, title_name, x, y, xlabel='Epochs'): if var_name not in self.plots: self.plots[var_name] = self.viz.line(X=np.array([x,x]), Y=np.array([y,y]), env=self.env, opts=dict( legend=[split_name], title=title_name, xlabel=xlabel, ylabel=var_name )) else: self.viz.line(X=np.array([x]), Y=np.array([y]), env=self.env, win=self.plots[var_name], name=split_name, update = 'append') def save_plots(self): windowstring = self.viz.get_window_data(env=self.env) windowdict = json.loads(windowstring) for windowname, windowcontent in windowdict.items(): data = [{k:datapoint[k] for k in ['x','y','type','name', 'mode']} for datapoint in windowcontent['content']['data']] layout = windowcontent['content']['layout'] plotlyfig = dict(data=data, layout=layout) plot_path = os.path.join(self.plot_path,windowcontent['title']+'.html') print("Saving plot to:", plot_path) plot(plotlyfig, filename=os.path.join(self.plot_path,windowcontent['title']+'.html'), auto_open=False)
def download_env(env): vis = Visdom('http://logserver.duckdns.org', port=5010) data = vis.get_window_data(env=env) d = json.loads(data) n_deleted = [] test_acc_avg = [] for key in d: try: #1 for MR 0 for UMICH x = list(d[key]["content"]["data"][1]["x"]) y = list(d[key]["content"]["data"][1]["y"]) if 'n-deleted' in key: n_deleted = (x, y) #1 for MR 0 for UMICH x = list(d[key]["content"]["data"][1]["x"]) y = list(d[key]["content"]["data"][1]["y"]) if 'test-acc-avg' in key: test_acc_avg = (x, y) except: pass return n_deleted, test_acc_avg
class VisdomLogger: def __init__(self, host='127.0.0.1', port=8097, env='main', log_path=None): from visdom import Visdom import json logger.info(f"using visdom on http://{host}:{port} env={env}") self.env = env self.viz = Visdom(server=f"http://{host}", port=port, env=env, log_to_filename=log_path) self.windows = dict() # if prev log exists if log_path.exists(): self.viz.replay_log(log_path) wins = json.loads(self.viz.get_window_data(win=None, env=env)) for k, v in wins.items(): names = [int(x['name']) for x in v['content']['data']] name = str(max(names) + 1) self.windows[v['title']] = {'win': v['id'], 'name': name} def add_plot(self, title, **kwargs): if title not in self.windows: self.windows[title] = { 'win': None, 'name': '1', } self.windows[title]['opts'] = { 'title': title, } self.windows[title]['opts'].update(kwargs) def add_point(self, title, x, y): X, Y = torch.FloatTensor([ x, ]), torch.FloatTensor([ y, ]) if title not in self.windows: self.add_plot(title) if self.windows[title]['win'] is None: w = self.viz.line(Y=Y, X=X, opts=self.windows[title]['opts'], name=self.windows[title]['name']) self.windows[title]['win'] = w else: self.viz.line(Y=Y, X=X, update='append', win=self.windows[title]['win'], name=self.windows[title]['name'])
def calc_ytick_range(vis: visdom.Visdom, window_name: str, env: Optional[str] = None) -> Tuple[float, float]: lower_bound, upper_bound = -1.0, 1.0 stats = vis.get_window_data(win=window_name, env=env) if stats: stats = json.loads(stats) stats = [np.array(item['y']) for item in stats['content']['data']] stats = [item[item != np.array([None])].astype(np.float16) for item in stats] if stats: q25s = np.array([np.quantile(item, 0.25) for item in stats if len(item) > 0]) q75s = np.array([np.quantile(item, 0.75) for item in stats if len(item) > 0]) if q25s.shape == q75s.shape and len(q25s) > 0: iqrs = q75s - q25s lower_bounds = q25s - 1.5 * iqrs upper_bounds = q75s + 1.5 * iqrs stats_sanitized = list() idx = 0 for item in stats: if len(item) > 0: item_sanitized = item[(item >= lower_bounds[idx]) & (item <= upper_bounds[idx])] stats_sanitized.append(item_sanitized) idx += 1 stats_sanitized = np.array(stats_sanitized) q25_sanitized = np.array([np.quantile(item, 0.25) for item in stats_sanitized]) q75_sanitized = np.array([np.quantile(item, 0.75) for item in stats_sanitized]) iqr_sanitized = np.sum(q75_sanitized - q25_sanitized) lower_bound = np.min(q25_sanitized) - 1.5 * iqr_sanitized upper_bound = np.max(q75_sanitized) + 1.5 * iqr_sanitized return lower_bound, upper_bound
class VisdomLogger: def __init__(self, host='127.0.0.1', port=8097, env='main', log_path=None, rank=None): from visdom import Visdom logger.debug(f"using visdom on http://{host}:{port} env={env}") self.env = env self.rank = rank self.viz = Visdom(server=f"http://{host}", port=port, env=env, log_to_filename=log_path) self.windows = dict() # if prev log exists if log_path is not None and log_path.exists() and (rank is None or rank == 0): self.viz.replay_log(log_path) def _get_win(self, title, type): import json win_data = json.loads(self.viz.get_window_data(win=None, env=self.env)) wins = [(w, v) for w, v in win_data.items() if v['title'] == title and v['type'] == type] if wins: handle, value = sorted(wins, key=lambda x: x[0])[0] return handle, value['content'] else: return None, None def _get_rank0_win(self, title, type): if self.rank is not None and self.rank > 0: # wait and fetch the window handle until rank=0 client generates new window for _ in range(10): handle, content = self._get_win(title, type) if handle is not None: return handle, content time.sleep(0.5) else: logger.error( "couldn't get a proper window handle from the visdom server" ) raise RuntimeError else: return self._get_win(title, type) def _new_window(self, cmd, title, **cmd_args): if cmd == self.viz.images: types = ("image", None) elif cmd == self.viz.scatter or cmd == self.viz.line: types = ("plot", "scatter") elif cmd == self.viz.heatmap: types = ("plot", "heatmap") else: types = ("plot", None) handle, content = self._get_rank0_win(title, types[0]) if handle is None: if "opts" in cmd_args: cmd_args['opts'].update({ "title": title, }) else: cmd_args['opts'] = { "title": title, } if types == ("plot", "scatter"): name = f"1_{self.rank}" if self.rank is not None else "1" handle = cmd(name=name, **cmd_args) else: name = None handle = cmd(**cmd_args) else: if types == ("plot", "scatter"): name = max([ int(x['name'].partition('_')[0]) for x in content['data'] ]) name = f"{name+1}_{self.rank}" if self.rank is not None else f"{name+1}" cmd(win=handle, name=name, update="append", **cmd_args) else: name = None handle = cmd(win=handle, **cmd_args) self.windows[title] = { 'handle': handle, 'name': name, 'opts': cmd_args["opts"], } def add_point(self, title, x, y, **kwargs): X, Y = torch.FloatTensor([ x, ]), torch.FloatTensor([ y, ]) if title not in self.windows: cmd = self.viz.line self._new_window(cmd, title, X=X, Y=Y, opts=kwargs) else: self.windows[title]['opts'].update(kwargs) handle = self.windows[title]['handle'] name = self.windows[title]['name'] opts = self.windows[title]['opts'] self.viz.line(win=handle, update='append', Y=Y, X=X, name=name, opts=opts) def plot_heatmap(self, title, tensor, **kwargs): if title not in self.windows: cmd = self.viz.heatmap self._new_window(cmd, title, X=tensor, opts=kwargs) else: self.windows[title]['opts'].update(kwargs) handle = self.windows[title]['handle'] opts = self.windows[title]['opts'] self.viz.heatmap(win=handle, X=tensor, opts=opts) def plot_images(self, title, tensor, nrow, **kwargs): if title not in self.windows: cmd = self.viz.images self._new_window(cmd, title, tensor=tensor, nrow=nrow, opts=kwargs) else: self.windows[title]['opts'].update(kwargs) handle = self.windows[title]['handle'] opts = self.windows[title]['opts'] self.viz.images(win=handle, tensor=tensor, nrow=nrow, opts=opts)
class Visualizer(object): """ Visualizer """ def __init__(self, port='13579', env='main', id=None): self.cur_win = {} self.vis = Visdom(port=port, env=env) self.id = id self.env = env # Restore ori_win = self.vis.get_window_data() ori_win = json.loads(ori_win) self.cur_win = {v['title']: k for k, v in ori_win.items()} def vis_scalar(self, win_name, trace_name, x, y, opts=None): """ Draw line """ if not isinstance(x, list): x = [x] if not isinstance(y, list): y = [y] if self.id is not None: win_name = "[%s]" % self.id + win_name default_opts = {'title': win_name} if opts is not None: default_opts.update(opts) win = self.cur_win.get(win_name, None) if win is not None: self.vis.line(X=x, Y=y, opts=default_opts, update='append', win=win, name=trace_name) else: self.cur_win[win_name] = self.vis.line(X=x, Y=y, opts=default_opts, name=trace_name) def vis_image(self, name, img, env=None, opts=None): """ vis image in visdom """ if env is None: env = self.env if self.id is not None: name = "[%s]" % self.id + name win = self.cur_win.get(name, None) default_opts = {'title': name} if opts is not None: default_opts.update(opts) if win is not None: self.vis.image(img=img, win=win, opts=opts, env=env) else: self.cur_win[name] = self.vis.image(img=img, opts=default_opts, env=env) def vis_table(self, name, tbl, opts=None): win = self.cur_win.get(name, None) tbl_str = "<table width=\"100%\"> " tbl_str += "<tr> \ <th>Term</th> \ <th>Value</th> \ </tr>" for k, v in tbl.items(): tbl_str += "<tr> \ <td>%s</td> \ <td>%s</td> \ </tr>" % (k, v) tbl_str += "</table>" default_opts = {'title': name} if opts is not None: default_opts.update(opts) if win is not None: self.vis.text(tbl_str, win=win, opts=default_opts) else: self.cur_win[name] = self.vis.text(tbl_str, opts=default_opts)
class Plot(object): def __init__(self, title="", env_name="", config=None, port=8080): self.env_name = env_name if env_name else title self.viz = Visdom(port=port, env=self.env_name) # self.viz.close() self.windows = {} self.title = title self.config = config def register_plot(self, name, xlabel, ylabel, plot_type="line", ymax=None): self.windows[name] = {"xlabel": xlabel, "ylabel": ylabel, "title": name, "plot_type": plot_type} self.windows[name]["opts"] = dict(title=name, markersize=5, xlabel=xlabel, ylabel=ylabel) if ymax is not None: self.windows[name]["opts"]["layoutopts"] = dict(plotly=dict(yaxis=dict(range=[0, ymax]))) def update_plot(self, plot_name, x, y, **kwargs): # Create plot if not registered try: plot_d = self.windows[plot_name] except: warnings.warn("Plot not found, creating new plot") plot_d = {"xlabel": "X", "ylabel": "Y", "plot_type": "scatter"} plotter = self.viz.scatter if plot_d["plot_type"] == "scatter" else self.viz.line # WHY WAS "Y" A NESTED LIST??? # data = {"X": np.asarray(x), "Y": np.asarray([y])} if plot_d["plot_type"] == "line" else {"X": np.asarray([x, y])} x = np.asarray(x) if len(x) < len(y): warnings.warn("X coords not found, interpolating") if x[0] == 0 and len(x) > 1: x[0] = x[1] - .001 additional_x = np.linspace(0, x[0], len(y) - len(x)) x = np.r_[additional_x, np.asarray(x)] data = {"X": x, "Y": np.asarray(y)} if plot_d["plot_type"] == "line" else {"X": np.asarray([x, y])} ## Update plot if "plot" in plot_d.keys(): plotter( **data, win=plot_d["plot"], update="append" ) else: # Create new plot win = plotter( **data, opts=plot_d["opts"], **kwargs ) plot_d["plot"] = win self.windows["name"] = plot_d # LOADING def load_log(self, path): self.viz.replay_log(path) def load_all_env(self, root, keyword="visdom"): for d, ss, fs in os.walk(root): for f in fs: full_env = os.path.join(d, f) # Don't load "BSF" graphs, just complete graphs if full_env[-5:] == ".json" and keyword in full_env and f != "losses.json" and "BSF_" not in full_env: print("Loading {}".format(full_env)) self.viz.replay_log(full_env) # viz.load load the environment to viz def save_env(self, file_path=None, current_env=None, new_env=None): if file_path is None: file_path = os.path.join(self.config["results_dir"], "visdom.json") if current_env is None: current_env = self.env_name new_env = current_env if new_env is None else new_env # self.viz = Visdom(env=current_env) # get current env data = json.loads(self.viz.get_window_data()) if len(data) == 0: print("NOTHING HAS BEEN SAVED: NOTHING IN THIS VISDOM ENV - DOES IT EXIST ?") return file = open(file_path, 'w+') for datapoint in data.values(): output = { 'win': datapoint['id'], 'eid': new_env, 'opts': {} } if datapoint['type'] != "plot": output['data'] = [{'content': datapoint['content'], 'type': datapoint['type']}] if datapoint['height'] is not None: output['opts']['height'] = datapoint['height'] if datapoint['width'] is not None: output['opts']['width'] = datapoint['width'] else: output['data'] = datapoint['content']["data"] output['layout'] = datapoint['content']["layout"] to_write = json.dumps(["events", output]) file.write(to_write + '\n') file.close()
def download_vse_sim(env, env2): vis = Visdom('http://logserver.duckdns.org', port=5010) path = '../results/{}___{}'.format(env, env2) pathlib.Path(path).mkdir(parents=True, exist_ok=True) x_axis = [] cap_recall_1 = [] cap_recall_2 = [] image_recall_1 = [] image_recall_2 = [] sum_1 = [] sum_2 = [] rsi = ['r1i', 'r5i', 'r10i'] rs = ['r1', 'r5', 'r10'] data = vis.get_window_data(env=env) d = json.loads(data) for key in d: try: x = list(d[key]["content"]["data"][0]["x"]) y = list(d[key]["content"]["data"][0]["y"]) if "/" in key and "avg_val" in key: match = key.split("/")[1] if match in rsi: image_recall_1.append((y, match)) x_axis = x elif match in rs: cap_recall_1.append((y, match)) elif 'sum' in key: sum_1 = (x, y) except: pass data = vis.get_window_data(env=env2) d = json.loads(data) for key in d: try: x = list(d[key]["content"]["data"][0]["x"]) y = list(d[key]["content"]["data"][0]["y"]) if "/" in key and "avg_val" in key: match = key.split("/")[1] if match in rsi: image_recall_2.append((y, match)) x_axis = x elif match in rs: cap_recall_2.append((y, match)) elif 'sum' in key: sum_2 = (x, y) except Exception as e: print(e) # Plot image recalls image_values_1 = [l[0] for l in image_recall_1] image_values_2 = [l[0] for l in image_recall_2] image_sum_1 = [sum(d) for d in zip(*image_values_1)] image_sum_2 = [sum(d) for d in zip(*image_values_2)] plots = [(image_sum_1, 'Random'), (image_sum_2, 'Intra caption')] plt.figure(1) plt.subplot(111) handles = [] for (line, label) in plots: handle, = plt.plot(x_axis, line, label=label) handles.append(handle) plt.legend(handles=handles) plt.xlabel('Amount of labeled samples') plt.ylabel('Image recall') plt.savefig('{}/image_recall.png'.format(path), dpi=600) # Plot caption recalls cap_values_1 = [l[0] for l in cap_recall_1] cap_values_2 = [l[0] for l in cap_recall_2] cap_sum_1 = [sum(d) for d in zip(*cap_values_1)] cap_sum_2 = [sum(d) for d in zip(*cap_values_2)] plots = [(cap_sum_1, 'Random'), (cap_sum_2, 'Intra caption')] plt.figure(2) plt.subplot(111) handles = [] for (line, label) in plots: handle, = plt.plot(x_axis, line, label=label) handles.append(handle) plt.legend(handles=handles) plt.xlabel('Amount of labeled samples') plt.ylabel('Caption recall') plt.savefig('{}/caption_recall.png'.format(path), dpi=600) # # Plot sum plots = [(sum_1[1], 'Random'), (sum_2[1], 'Intra caption')] plt.figure(3) plt.subplot(111) handles = [] x_axis = sum_1[0] for (line, label) in plots: handle, = plt.plot(x_axis, line, label=label) handles.append(handle) plt.legend(handles=handles) plt.xlabel('Amount of labeled samples') plt.ylabel('Sum of recalls') plt.savefig('{}/sum.png'.format(path), dpi=600)
def download_vse_env(env): vis = Visdom('http://logserver.duckdns.org', port=5010) data = vis.get_window_data(env=env) d = json.loads(data) path = '../results/{}'.format(env) pathlib.Path(path).mkdir(parents=True, exist_ok=True) x_axis = [] sum = [] loss = [] num_0_actions = [] cap_recall = [] image_recall = [] rsi = ['r1i', 'r5i', 'r10i'] rs = ['r1', 'r5', 'r10'] for key in d: try: x = list(d[key]["content"]["data"][0]["x"]) y = list(d[key]["content"]["data"][0]["y"]) if "/" in key: match = key.split("/")[1] if match in rsi: image_recall.append((y, match)) x_axis = x elif match in rs: cap_recall.append((y, match)) elif 'sum' in key: sum = (x, y) elif 'actions' in key: num_0_actions = (x, y) loss = (x, y) except: pass # cap_recall = sorted(cap_recall, key=lambda x: x[1]) # cap_recall = sorted(cap_recall, key=lambda x: x[1]) # Plot image recalls plt.figure(1) plt.subplot(111) handles = [] for (line, label) in image_recall: handle, = plt.plot(x_axis, line, label=label) handles.append(handle) plt.legend(handles=handles) plt.xlabel('Episodes') plt.ylabel('Image recall') plt.savefig('{}/image_recall.png'.format(env), dpi=600) # Plot caption recalls plt.figure(2) plt.subplot(111) handles = [] for (line, label) in cap_recall: handle, = plt.plot(x_axis, line, label=label) handles.append(handle) plt.legend(handles=handles) plt.xlabel('Episodes') plt.ylabel('Caption recall') plt.savefig('{}/caption_recall.png'.format(path, env), dpi=600) # Plot sum plt.figure(3) plt.subplot(111) plt.plot(*sum) plt.xlabel('Episodes') plt.ylabel('Sum of recalls') plt.savefig('{}/sum.png'.format(path, env), dpi=600) # Plot loss plt.figure(4) plt.subplot(111) plt.plot(*loss) plt.xlabel('Episodes') plt.ylabel('Loss') plt.savefig('{}/loss.png'.format(path, env), dpi=600) # Plot sum plt.figure(5) plt.subplot(111) plt.plot(*num_0_actions) plt.xlabel('Episodes') plt.ylabel('Number of 0 actions') plt.savefig('{}/num_0_actions.png'.format(path, env), dpi=600)