class VisdomLinePlotter(object):
"""Plots to Visdom"""
def __init__(self, env_name='main', plot_path='/tmp'):
self.viz = Visdom(port=6065)
self.env = env_name
self.plots = {}
self.plot_path = plot_path
#self.viz.log_to_filename = log_filename
def plot(self, var_name, split_name, title_name, x, y, xlabel='Epochs'):
if var_name not in self.plots:
self.plots[var_name] = self.viz.line(X=np.array([x,x]), Y=np.array([y,y]), env=self.env, opts=dict(
legend=[split_name],
title=title_name,
xlabel=xlabel,
ylabel=var_name
))
else:
self.viz.line(X=np.array([x]), Y=np.array([y]), env=self.env, win=self.plots[var_name], name=split_name, update = 'append')
def save_plots(self):
windowstring = self.viz.get_window_data(env=self.env)
windowdict = json.loads(windowstring)
for windowname, windowcontent in windowdict.items():
data = [{k:datapoint[k] for k in ['x','y','type','name', 'mode']} for datapoint in windowcontent['content']['data']]
layout = windowcontent['content']['layout']
plotlyfig = dict(data=data, layout=layout)
plot_path = os.path.join(self.plot_path,windowcontent['title']+'.html')
print("Saving plot to:", plot_path)
plot(plotlyfig, filename=os.path.join(self.plot_path,windowcontent['title']+'.html'), auto_open=False)
def download_env(env):
vis = Visdom('http://logserver.duckdns.org', port=5010)
data = vis.get_window_data(env=env)
d = json.loads(data)
n_deleted = []
test_acc_avg = []
for key in d:
try:
#1 for MR 0 for UMICH
x = list(d[key]["content"]["data"][1]["x"])
y = list(d[key]["content"]["data"][1]["y"])
if 'n-deleted' in key:
n_deleted = (x, y)
#1 for MR 0 for UMICH
x = list(d[key]["content"]["data"][1]["x"])
y = list(d[key]["content"]["data"][1]["y"])
if 'test-acc-avg' in key:
test_acc_avg = (x, y)
except:
pass
return n_deleted, test_acc_avg
class VisdomLogger:
def __init__(self, host='127.0.0.1', port=8097, env='main', log_path=None):
from visdom import Visdom
import json
logger.info(f"using visdom on http://{host}:{port} env={env}")
self.env = env
self.viz = Visdom(server=f"http://{host}",
port=port,
env=env,
log_to_filename=log_path)
self.windows = dict()
# if prev log exists
if log_path.exists():
self.viz.replay_log(log_path)
wins = json.loads(self.viz.get_window_data(win=None, env=env))
for k, v in wins.items():
names = [int(x['name']) for x in v['content']['data']]
name = str(max(names) + 1)
self.windows[v['title']] = {'win': v['id'], 'name': name}
def add_plot(self, title, **kwargs):
if title not in self.windows:
self.windows[title] = {
'win': None,
'name': '1',
}
self.windows[title]['opts'] = {
'title': title,
}
self.windows[title]['opts'].update(kwargs)
def add_point(self, title, x, y):
X, Y = torch.FloatTensor([
x,
]), torch.FloatTensor([
y,
])
if title not in self.windows:
self.add_plot(title)
if self.windows[title]['win'] is None:
w = self.viz.line(Y=Y,
X=X,
opts=self.windows[title]['opts'],
name=self.windows[title]['name'])
self.windows[title]['win'] = w
else:
self.viz.line(Y=Y,
X=X,
update='append',
win=self.windows[title]['win'],
name=self.windows[title]['name'])
def calc_ytick_range(vis: visdom.Visdom, window_name: str, env: Optional[str] = None) -> Tuple[float, float]:
lower_bound, upper_bound = -1.0, 1.0
stats = vis.get_window_data(win=window_name, env=env)
if stats:
stats = json.loads(stats)
stats = [np.array(item['y']) for item in stats['content']['data']]
stats = [item[item != np.array([None])].astype(np.float16) for item in stats]
if stats:
q25s = np.array([np.quantile(item, 0.25) for item in stats if len(item) > 0])
q75s = np.array([np.quantile(item, 0.75) for item in stats if len(item) > 0])
if q25s.shape == q75s.shape and len(q25s) > 0:
iqrs = q75s - q25s
lower_bounds = q25s - 1.5 * iqrs
upper_bounds = q75s + 1.5 * iqrs
stats_sanitized = list()
idx = 0
for item in stats:
if len(item) > 0:
item_sanitized = item[(item >= lower_bounds[idx]) & (item <= upper_bounds[idx])]
stats_sanitized.append(item_sanitized)
idx += 1
stats_sanitized = np.array(stats_sanitized)
q25_sanitized = np.array([np.quantile(item, 0.25) for item in stats_sanitized])
q75_sanitized = np.array([np.quantile(item, 0.75) for item in stats_sanitized])
iqr_sanitized = np.sum(q75_sanitized - q25_sanitized)
lower_bound = np.min(q25_sanitized) - 1.5 * iqr_sanitized
upper_bound = np.max(q75_sanitized) + 1.5 * iqr_sanitized
return lower_bound, upper_bound
class VisdomLogger:
def __init__(self,
host='127.0.0.1',
port=8097,
env='main',
log_path=None,
rank=None):
from visdom import Visdom
logger.debug(f"using visdom on http://{host}:{port} env={env}")
self.env = env
self.rank = rank
self.viz = Visdom(server=f"http://{host}",
port=port,
env=env,
log_to_filename=log_path)
self.windows = dict()
# if prev log exists
if log_path is not None and log_path.exists() and (rank is None
or rank == 0):
self.viz.replay_log(log_path)
def _get_win(self, title, type):
import json
win_data = json.loads(self.viz.get_window_data(win=None, env=self.env))
wins = [(w, v) for w, v in win_data.items()
if v['title'] == title and v['type'] == type]
if wins:
handle, value = sorted(wins, key=lambda x: x[0])[0]
return handle, value['content']
else:
return None, None
def _get_rank0_win(self, title, type):
if self.rank is not None and self.rank > 0:
# wait and fetch the window handle until rank=0 client generates new window
for _ in range(10):
handle, content = self._get_win(title, type)
if handle is not None:
return handle, content
time.sleep(0.5)
else:
logger.error(
"couldn't get a proper window handle from the visdom server"
)
raise RuntimeError
else:
return self._get_win(title, type)
def _new_window(self, cmd, title, **cmd_args):
if cmd == self.viz.images:
types = ("image", None)
elif cmd == self.viz.scatter or cmd == self.viz.line:
types = ("plot", "scatter")
elif cmd == self.viz.heatmap:
types = ("plot", "heatmap")
else:
types = ("plot", None)
handle, content = self._get_rank0_win(title, types[0])
if handle is None:
if "opts" in cmd_args:
cmd_args['opts'].update({
"title": title,
})
else:
cmd_args['opts'] = {
"title": title,
}
if types == ("plot", "scatter"):
name = f"1_{self.rank}" if self.rank is not None else "1"
handle = cmd(name=name, **cmd_args)
else:
name = None
handle = cmd(**cmd_args)
else:
if types == ("plot", "scatter"):
name = max([
int(x['name'].partition('_')[0]) for x in content['data']
])
name = f"{name+1}_{self.rank}" if self.rank is not None else f"{name+1}"
cmd(win=handle, name=name, update="append", **cmd_args)
else:
name = None
handle = cmd(win=handle, **cmd_args)
self.windows[title] = {
'handle': handle,
'name': name,
'opts': cmd_args["opts"],
}
def add_point(self, title, x, y, **kwargs):
X, Y = torch.FloatTensor([
x,
]), torch.FloatTensor([
y,
])
if title not in self.windows:
cmd = self.viz.line
self._new_window(cmd, title, X=X, Y=Y, opts=kwargs)
else:
self.windows[title]['opts'].update(kwargs)
handle = self.windows[title]['handle']
name = self.windows[title]['name']
opts = self.windows[title]['opts']
self.viz.line(win=handle,
update='append',
Y=Y,
X=X,
name=name,
opts=opts)
def plot_heatmap(self, title, tensor, **kwargs):
if title not in self.windows:
cmd = self.viz.heatmap
self._new_window(cmd, title, X=tensor, opts=kwargs)
else:
self.windows[title]['opts'].update(kwargs)
handle = self.windows[title]['handle']
opts = self.windows[title]['opts']
self.viz.heatmap(win=handle, X=tensor, opts=opts)
def plot_images(self, title, tensor, nrow, **kwargs):
if title not in self.windows:
cmd = self.viz.images
self._new_window(cmd, title, tensor=tensor, nrow=nrow, opts=kwargs)
else:
self.windows[title]['opts'].update(kwargs)
handle = self.windows[title]['handle']
opts = self.windows[title]['opts']
self.viz.images(win=handle, tensor=tensor, nrow=nrow, opts=opts)
class Visualizer(object):
""" Visualizer
"""
def __init__(self, port='13579', env='main', id=None):
self.cur_win = {}
self.vis = Visdom(port=port, env=env)
self.id = id
self.env = env
# Restore
ori_win = self.vis.get_window_data()
ori_win = json.loads(ori_win)
self.cur_win = {v['title']: k for k, v in ori_win.items()}
def vis_scalar(self, win_name, trace_name, x, y, opts=None):
""" Draw line
"""
if not isinstance(x, list):
x = [x]
if not isinstance(y, list):
y = [y]
if self.id is not None:
win_name = "[%s]" % self.id + win_name
default_opts = {'title': win_name}
if opts is not None:
default_opts.update(opts)
win = self.cur_win.get(win_name, None)
if win is not None:
self.vis.line(X=x,
Y=y,
opts=default_opts,
update='append',
win=win,
name=trace_name)
else:
self.cur_win[win_name] = self.vis.line(X=x,
Y=y,
opts=default_opts,
name=trace_name)
def vis_image(self, name, img, env=None, opts=None):
""" vis image in visdom
"""
if env is None:
env = self.env
if self.id is not None:
name = "[%s]" % self.id + name
win = self.cur_win.get(name, None)
default_opts = {'title': name}
if opts is not None:
default_opts.update(opts)
if win is not None:
self.vis.image(img=img, win=win, opts=opts, env=env)
else:
self.cur_win[name] = self.vis.image(img=img,
opts=default_opts,
env=env)
def vis_table(self, name, tbl, opts=None):
win = self.cur_win.get(name, None)
tbl_str = "<table width=\"100%\"> "
tbl_str += "<tr> \
<th>Term</th> \
<th>Value</th> \
</tr>"
for k, v in tbl.items():
tbl_str += "<tr> \
<td>%s</td> \
<td>%s</td> \
</tr>" % (k, v)
tbl_str += "</table>"
default_opts = {'title': name}
if opts is not None:
default_opts.update(opts)
if win is not None:
self.vis.text(tbl_str, win=win, opts=default_opts)
else:
self.cur_win[name] = self.vis.text(tbl_str, opts=default_opts)
class Plot(object):
def __init__(self, title="", env_name="", config=None, port=8080):
self.env_name = env_name if env_name else title
self.viz = Visdom(port=port, env=self.env_name)
# self.viz.close()
self.windows = {}
self.title = title
self.config = config
def register_plot(self, name, xlabel, ylabel, plot_type="line", ymax=None):
self.windows[name] = {"xlabel": xlabel, "ylabel": ylabel, "title": name, "plot_type": plot_type}
self.windows[name]["opts"] = dict(title=name, markersize=5, xlabel=xlabel, ylabel=ylabel)
if ymax is not None:
self.windows[name]["opts"]["layoutopts"] = dict(plotly=dict(yaxis=dict(range=[0, ymax])))
def update_plot(self, plot_name, x, y, **kwargs):
# Create plot if not registered
try:
plot_d = self.windows[plot_name]
except:
warnings.warn("Plot not found, creating new plot")
plot_d = {"xlabel": "X", "ylabel": "Y", "plot_type": "scatter"}
plotter = self.viz.scatter if plot_d["plot_type"] == "scatter" else self.viz.line
# WHY WAS "Y" A NESTED LIST???
# data = {"X": np.asarray(x), "Y": np.asarray([y])} if plot_d["plot_type"] == "line" else {"X": np.asarray([x, y])}
x = np.asarray(x)
if len(x) < len(y):
warnings.warn("X coords not found, interpolating")
if x[0] == 0 and len(x) > 1:
x[0] = x[1] - .001
additional_x = np.linspace(0, x[0], len(y) - len(x))
x = np.r_[additional_x, np.asarray(x)]
data = {"X": x, "Y": np.asarray(y)} if plot_d["plot_type"] == "line" else {"X": np.asarray([x, y])}
## Update plot
if "plot" in plot_d.keys():
plotter(
**data,
win=plot_d["plot"],
update="append"
)
else: # Create new plot
win = plotter(
**data,
opts=plot_d["opts"], **kwargs
)
plot_d["plot"] = win
self.windows["name"] = plot_d
# LOADING
def load_log(self, path):
self.viz.replay_log(path)
def load_all_env(self, root, keyword="visdom"):
for d, ss, fs in os.walk(root):
for f in fs:
full_env = os.path.join(d, f)
# Don't load "BSF" graphs, just complete graphs
if full_env[-5:] == ".json" and keyword in full_env and f != "losses.json" and "BSF_" not in full_env:
print("Loading {}".format(full_env))
self.viz.replay_log(full_env) # viz.load load the environment to viz
def save_env(self, file_path=None, current_env=None, new_env=None):
if file_path is None:
file_path = os.path.join(self.config["results_dir"], "visdom.json")
if current_env is None:
current_env = self.env_name
new_env = current_env if new_env is None else new_env
# self.viz = Visdom(env=current_env) # get current env
data = json.loads(self.viz.get_window_data())
if len(data) == 0:
print("NOTHING HAS BEEN SAVED: NOTHING IN THIS VISDOM ENV - DOES IT EXIST ?")
return
file = open(file_path, 'w+')
for datapoint in data.values():
output = {
'win': datapoint['id'],
'eid': new_env,
'opts': {}
}
if datapoint['type'] != "plot":
output['data'] = [{'content': datapoint['content'], 'type': datapoint['type']}]
if datapoint['height'] is not None:
output['opts']['height'] = datapoint['height']
if datapoint['width'] is not None:
output['opts']['width'] = datapoint['width']
else:
output['data'] = datapoint['content']["data"]
output['layout'] = datapoint['content']["layout"]
to_write = json.dumps(["events", output])
file.write(to_write + '\n')
file.close()
def download_vse_sim(env, env2):
vis = Visdom('http://logserver.duckdns.org', port=5010)
path = '../results/{}___{}'.format(env, env2)
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
x_axis = []
cap_recall_1 = []
cap_recall_2 = []
image_recall_1 = []
image_recall_2 = []
sum_1 = []
sum_2 = []
rsi = ['r1i', 'r5i', 'r10i']
rs = ['r1', 'r5', 'r10']
data = vis.get_window_data(env=env)
d = json.loads(data)
for key in d:
try:
x = list(d[key]["content"]["data"][0]["x"])
y = list(d[key]["content"]["data"][0]["y"])
if "/" in key and "avg_val" in key:
match = key.split("/")[1]
if match in rsi:
image_recall_1.append((y, match))
x_axis = x
elif match in rs:
cap_recall_1.append((y, match))
elif 'sum' in key:
sum_1 = (x, y)
except:
pass
data = vis.get_window_data(env=env2)
d = json.loads(data)
for key in d:
try:
x = list(d[key]["content"]["data"][0]["x"])
y = list(d[key]["content"]["data"][0]["y"])
if "/" in key and "avg_val" in key:
match = key.split("/")[1]
if match in rsi:
image_recall_2.append((y, match))
x_axis = x
elif match in rs:
cap_recall_2.append((y, match))
elif 'sum' in key:
sum_2 = (x, y)
except Exception as e:
print(e)
# Plot image recalls
image_values_1 = [l[0] for l in image_recall_1]
image_values_2 = [l[0] for l in image_recall_2]
image_sum_1 = [sum(d) for d in zip(*image_values_1)]
image_sum_2 = [sum(d) for d in zip(*image_values_2)]
plots = [(image_sum_1, 'Random'), (image_sum_2, 'Intra caption')]
plt.figure(1)
plt.subplot(111)
handles = []
for (line, label) in plots:
handle, = plt.plot(x_axis, line, label=label)
handles.append(handle)
plt.legend(handles=handles)
plt.xlabel('Amount of labeled samples')
plt.ylabel('Image recall')
plt.savefig('{}/image_recall.png'.format(path), dpi=600)
# Plot caption recalls
cap_values_1 = [l[0] for l in cap_recall_1]
cap_values_2 = [l[0] for l in cap_recall_2]
cap_sum_1 = [sum(d) for d in zip(*cap_values_1)]
cap_sum_2 = [sum(d) for d in zip(*cap_values_2)]
plots = [(cap_sum_1, 'Random'), (cap_sum_2, 'Intra caption')]
plt.figure(2)
plt.subplot(111)
handles = []
for (line, label) in plots:
handle, = plt.plot(x_axis, line, label=label)
handles.append(handle)
plt.legend(handles=handles)
plt.xlabel('Amount of labeled samples')
plt.ylabel('Caption recall')
plt.savefig('{}/caption_recall.png'.format(path), dpi=600)
# # Plot sum
plots = [(sum_1[1], 'Random'), (sum_2[1], 'Intra caption')]
plt.figure(3)
plt.subplot(111)
handles = []
x_axis = sum_1[0]
for (line, label) in plots:
handle, = plt.plot(x_axis, line, label=label)
handles.append(handle)
plt.legend(handles=handles)
plt.xlabel('Amount of labeled samples')
plt.ylabel('Sum of recalls')
plt.savefig('{}/sum.png'.format(path), dpi=600)
def download_vse_env(env):
vis = Visdom('http://logserver.duckdns.org', port=5010)
data = vis.get_window_data(env=env)
d = json.loads(data)
path = '../results/{}'.format(env)
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
x_axis = []
sum = []
loss = []
num_0_actions = []
cap_recall = []
image_recall = []
rsi = ['r1i', 'r5i', 'r10i']
rs = ['r1', 'r5', 'r10']
for key in d:
try:
x = list(d[key]["content"]["data"][0]["x"])
y = list(d[key]["content"]["data"][0]["y"])
if "/" in key:
match = key.split("/")[1]
if match in rsi:
image_recall.append((y, match))
x_axis = x
elif match in rs:
cap_recall.append((y, match))
elif 'sum' in key:
sum = (x, y)
elif 'actions' in key:
num_0_actions = (x, y)
loss = (x, y)
except:
pass
# cap_recall = sorted(cap_recall, key=lambda x: x[1])
# cap_recall = sorted(cap_recall, key=lambda x: x[1])
# Plot image recalls
plt.figure(1)
plt.subplot(111)
handles = []
for (line, label) in image_recall:
handle, = plt.plot(x_axis, line, label=label)
handles.append(handle)
plt.legend(handles=handles)
plt.xlabel('Episodes')
plt.ylabel('Image recall')
plt.savefig('{}/image_recall.png'.format(env), dpi=600)
# Plot caption recalls
plt.figure(2)
plt.subplot(111)
handles = []
for (line, label) in cap_recall:
handle, = plt.plot(x_axis, line, label=label)
handles.append(handle)
plt.legend(handles=handles)
plt.xlabel('Episodes')
plt.ylabel('Caption recall')
plt.savefig('{}/caption_recall.png'.format(path, env), dpi=600)
# Plot sum
plt.figure(3)
plt.subplot(111)
plt.plot(*sum)
plt.xlabel('Episodes')
plt.ylabel('Sum of recalls')
plt.savefig('{}/sum.png'.format(path, env), dpi=600)
# Plot loss
plt.figure(4)
plt.subplot(111)
plt.plot(*loss)
plt.xlabel('Episodes')
plt.ylabel('Loss')
plt.savefig('{}/loss.png'.format(path, env), dpi=600)
# Plot sum
plt.figure(5)
plt.subplot(111)
plt.plot(*num_0_actions)
plt.xlabel('Episodes')
plt.ylabel('Number of 0 actions')
plt.savefig('{}/num_0_actions.png'.format(path, env), dpi=600)
