def __init__(self, server='localhost', port=8097):
self.server = server
self.port = port
try:
vis = Visdom(server="http://"+self.server, port=self.port)
vis.win_exists("test")#dummy call to make sure that a connection to
# visdom can be established
except:
print("\n\n\TortillaError :: Unable to connect to Visdom Server even when you have plotting on. \n\
Are you sure that you have visdom server running at : \
http://{}:{} ? \n \
If not, please start visdom by running : \n\npython -m visdom.server \n\n\n \
Or...disable plotting by passing the --no-plots option. \
".format(server,port))
exit(0)
class BaseVis(object):
def __init__(self,
viz_opts,
update_mode='append',
env=None,
win=None,
resume=False,
port=8097):
self.viz_opts = viz_opts
self.update_mode = update_mode
self.win = win
if env is None:
env = 'main'
self.viz = Visdom(env=env, port=port)
# if resume first plot should not update with replace
self.removed = not resume
def win_exists(self):
return self.viz.win_exists(self.win)
def close(self):
if self.win is not None:
self.viz.close(win=self.win)
self.win = None
def register_event_handler(self, handler):
self.viz.register_event_handler(handler, self.win)
def create_summary_window(vis: visdom.Visdom,
visdom_env_name: str,
experiment_name: str,
summary: str) -> str:
return vis.text(
text=summary,
win=experiment_name,
env=visdom_env_name,
opts={'title': 'Summary', 'width': 576, 'height': 416},
append=vis.win_exists(experiment_name, visdom_env_name)
)
),
)
win = viz.scatter(
X=np.random.rand(255, 2),
opts=dict(
markersize=10,
markercolor=np.random.randint(0, 255, (
255,
3,
)),
),
)
# assert that the window exists
assert viz.win_exists(win), 'Created window marked as not existing'
# add new trace to scatter plot
viz.scatter(X=np.random.rand(255),
Y=np.random.rand(255),
win=win,
name='new_trace',
update='new')
# 2D scatter plot with text labels:
viz.scatter(X=np.random.rand(10, 2),
opts=dict(textlabels=['Label %d' % (i + 1)
for i in range(10)]))
viz.scatter(X=np.random.rand(10, 2),
Y=[1] * 5 + [2] * 3 + [3] * 2,
opts=dict(legend=['A', 'B', 'C'],
),
)
win = viz.scatter(
X=np.random.rand(255, 2),
opts=dict(
markersize=10,
markercolor=np.random.randint(0, 255, (
255,
3,
)),
),
)
# assert that the window exists
assert viz.win_exists(win)
# add new trace to scatter plot
viz.updateTrace(
X=np.random.rand(255),
Y=np.random.rand(255),
win=win,
name='new_trace',
)
# bar plots
viz.bar(X=np.random.rand(20))
viz.bar(X=np.abs(np.random.rand(5, 3)),
opts=dict(stacked=True,
legend=['Facebook', 'Google', 'Twitter'],
rownames=['2012', '2013', '2014', '2015', '2016']))
class Visualizer(object):
def __init__(self, config: Config):
# logging_level = logging._checkLevel("INFO")
# logging.getLogger().setLevel(logging_level)
# VisdomServer.start_server(port=VisdomServer.DEFAULT_PORT, env_path=config.vis_env_path)
self.reinit(config)
def reinit(self, config):
self.config = config
try:
self.visdom = Visdom(env=config.visdom_env)
self.connected = self.visdom.check_connection()
if not self.connected:
print(
"Visdom server hasn't started, please run command 'python -m visdom.server' in terminal."
)
# try:
# print("Visdom server hasn't started, do you want to start it? ")
# if 'y' in input("y/n: ").lower():
# os.popen('python -m visdom.server')
# except Exception as e:
# warn(e)
except ConnectionError as e:
warn("Can't open Visdom because " + e.strerror)
with open(self.config.log_file, 'a') as f:
info = "[{time}]Initialize Visdom\n".format(
time=timestr('%m-%d %H:%M:%S'))
info += str(self.config)
f.write(info + '\n')
def save(self, save_path: str = None) -> str:
retstr = self.visdom.save([
self.config.visdom_env
]) # return current environments name in format of json
try:
ret = json.loads(retstr)[0]
if ret == self.config.visdom_env:
if isinstance(save_path, str):
from shutil import copy
copy(self.config.vis_env_path, save_path)
print('Visdom Environment has saved into ' + save_path)
else:
print('Visdom Environment has saved into ' +
self.config.vis_env_path)
with open(self.config.vis_env_path, 'r') as fp:
env_str = json.load(fp)
return env_str
except Exception as e:
warn(e)
return None
def clear(self):
self.visdom.close()
@staticmethod
def _to_numpy(value):
if isinstance(value, t.Tensor):
value = value.cpu().detach().numpy()
elif isinstance(value, np.ndarray):
pass
else:
value = np.array(value)
if value.ndim == 0:
value = value[np.newaxis]
return value
def plot(self, y, x, line_name, win, legend=None):
# type:(float,float,str,str,list)->bool
"""Plot a (sequence) of y point(s) (each) with one x value(s), loop this method to draw whole plot"""
update = None if not self.visdom.win_exists(win) else 'append'
opts = dict(title=win)
if legend is not None:
opts["legend"] = legend
y = Visualizer._to_numpy(y)
x = Visualizer._to_numpy(x)
return win == self.visdom.line(y,
x,
win=win,
env=self.config.visdom_env,
update=update,
name=line_name,
opts=opts)
def bar(self, y, win, rowindices=None):
opts = dict(title=win)
y = Visualizer._to_numpy(y)
if isinstance(rowindices, list) and len(rowindices) == len(y):
opts["rownames"] = rowindices
return win == self.visdom.bar(y,
win=win,
env=self.config.visdom_env,
opts=opts)
def log(self, msg, name, append=True, log_file=None):
# type:(Visualizer,str,str,bool,str)->bool
if log_file is None:
log_file = self.config.log_file
info = "[{time}]{msg}".format(time=timestr('%m-%d %H:%M:%S'), msg=msg)
append = append and self.visdom.win_exists(name)
ret = self.visdom.text(info,
win=name,
env=self.config.visdom_env,
opts=dict(title=name),
append=append)
mode = 'a+' if append else 'w+'
with open(log_file, mode) as f:
f.write(info + '\n')
return ret == name
def log_process(self, num, total, msg, name, append=True):
# type:(Visualizer,int,int,str,str,bool)->bool
info = "[{time}]{msg}".format(time=timestr('%m-%d %H:%M:%S'), msg=msg)
append = append and self.visdom.win_exists(name)
ret = self.visdom.text(info,
win=(name),
env=self.config.visdom_env,
opts=dict(title=name),
append=append)
with open(self.config.log_file, 'a') as f:
f.write(info + '\n')
self.process_bar(num, total, msg)
return ret == name
def process_bar(self, num, total, msg='', length=50):
rate = num / total
rate_num = int(rate * 100)
clth = int(rate * length)
if len(msg) > 0:
msg += ':'
# msg = msg.replace('\n', '').replace('\r', '')
if rate_num == 100:
r = '\r%s[%s%d%%]\n' % (
msg,
'*' * length,
rate_num,
)
else:
r = '\r%s[%s%s%d%%]' % (
msg,
'*' * clth,
'-' * (length - clth),
rate_num,
)
sys.stdout.write(r)
sys.stdout.flush()
return r.replace('\r', ':')
def plot_line(vis: visdom.Visdom,
window_name: str,
env: Optional[str] = None,
line_label: Optional[str] = None,
x: Optional[np.ndarray] = None,
y: Optional[np.ndarray] = None,
x_label: Optional[str] = None,
y_label: Optional[str] = None,
width: int = 576,
height: int = 416,
draw_marker: bool = False) -> str:
empty_call = not vis.win_exists(window_name)
if empty_call and (x is not None or y is not None):
return window_name
if x is None:
x = np.ones(1)
empty_call = empty_call & True
if y is None:
y = np.full(1, np.nan)
empty_call = empty_call & True
if x.shape != y.shape:
x = np.ones_like(y)
opts = {
'showlegend': True,
'markers': draw_marker,
'markersize': 5,
}
if empty_call:
opts['title'] = window_name
opts['width'] = width
opts['height'] = height
window_name = vis.line(
X=x,
Y=y,
win=window_name,
env=env,
update='append',
name=line_label,
opts=opts
)
xtickmin, xtickmax = 0.0, np.max(x) * 1.05
ytickmin, ytickmax = calc_ytick_range(vis, window_name, env)
opts = {
'showlegend': True,
'xtickmin': xtickmin,
'xtickmax': xtickmax,
'ytickmin': ytickmin,
'ytickmax': ytickmax,
'xlabel': x_label,
'ylabel': y_label
}
window_name = vis.update_window_opts(win=window_name, opts=opts, env=env)
return window_name
def build_logger(build_state, checkpoint={}, run='NavA3C', port=8097):
vis = Visdom(port=port)
env = run
wins = mp.Manager().dict()
offset = checkpoint.setdefault('offset', -1) + 1
if 'plots' in checkpoint:
wins = dict((name, id) for name, id in checkpoint['plots'].items()
if vis.win_exists(id, env))
def _save_checkpoint(step):
if step % args.save_interval != 0 or args.checkpoint_path is None:
return
checkpoint_path = os.path.abspath(args.checkpoint_path)
checkpoint_dir = os.path.dirname(checkpoint_path)
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
state = build_state()
state['plots'] = dict(wins)
state['offset'] = offset
torch.save(state, args.checkpoint_path)
def _log_scatter(value, step, win_name, title, mode='train'):
if mode == 'test':
step += offset
elif step % args.log_interval != 0:
return
if not vis.check_connection():
return
if isinstance(value, torch.Tensor):
norm = value.numpy()
else:
norm = value
win_id = wins.setdefault(win_name)
if win_id is None:
wins[win_name] = vis.scatter(X=np.array([[step, norm]]),
win=win_id,
env=env,
opts=dict(title=title))
else:
vis.scatter(X=np.array([[step, norm]]),
win=win_id,
env=env,
update='append')
vis.save([env])
def _log_reward(total_reward, step, mode):
if mode == 'train' and step % args.log_interval != 0:
return
if not vis.check_connection():
return
win_name = 'total_reward_{}'.format(mode)
win_id = wins.setdefault(win_name)
if win_id is None:
wins[win_name] = vis.line(
np.array([0, 0]),
win=win_id,
env=env,
opts=dict(title='{} reward'.format(mode)))
else:
vis.line(Y=np.array([total_reward]),
X=np.array([step]),
win=win_id,
env=env,
update='append')
vis.save([env])
def _log_video(video, step):
step += offset
if args.video_path is None:
return
video_path = os.path.abspath(args.video_path)
video_dir = os.path.dirname(video_path)
if not os.path.exists(video_dir):
os.makedirs(video_dir)
skvideo.io.vwrite(video_path, np.array(video))
if not vis.check_connection():
return
win_name = 'last_test_episode'
win_id = wins.setdefault(win_name)
if win_id is None:
wins[win_name] = vis.video(
videofile=video_path,
win=win_id,
env=env,
opts=dict(title='episode {}'.format(step)))
else:
vis.video(videofile=video_path,
win=win_id,
env=env,
opts=dict(title='episode {}'.format(step)))
vis.save([env])
return dict(
video=_log_video,
grad_norm=lambda n, s: _log_scatter(n, s, 'grad_norm', 'gradient norm'
),
train_reward=lambda r, s: _log_reward(r, s, 'train'),
test_reward=lambda r, s: _log_reward(r, s, 'test'),
train_time=lambda n, s: _log_scatter(
n, s, 'train_time', 'training wall time (per episode)', 'train'),
test_time=lambda n, s: _log_scatter(
n, s, 'test_time', 'evaluation wall time (per episode)', 'test'),
checkpoint=_save_checkpoint)
def align_execute_mean_two():
temp = []
f = open(two_task_time_csv, 'r')
csv_r = csv.reader(f)
for p in csv_r:
temp.append(p)
f.close()
length = len(temp)
print('length of rough pearson correlation list: %d' % length)
# Build the dictionary for storing the complemented correlation
str_i = '-'
inference_exe_join = []
for i in inference_exe:
# inference_exe_join.append(str_i.join(i))
inference_exe_join.append(i[2] + '_' + i[3] + '_' + i[1])
dict_two = {}
# em_exe = poly_exe + inference_exe_join
em_exe = poly_exe + inference_exe_join + example_exe
for p in em_exe:
dict_two[p] = []
# search each kind of benchmark and complement all the combinations with it
for i in range(length):
obj = temp[i]
print('*** %s *** & *** %s ***' % (obj[0], obj[1]))
group = temp[i]
if group[0] != group[1]:
x1, x2 = dict_two[group[0]], dict_two[group[1]]
x1.append(group)
x2.append(
[group[1], group[0], group[4], group[5], group[2], group[3]])
dict_two[group[0]], dict_two[group[1]] = x1, x2
else:
x1 = dict_two[group[0]]
x1.append(group)
dict_two[group[0]] = x1
# write the dictionary to csv file
f = open(align_two_task_time_csv, 'a+')
csv_w = csv.writer(f)
for p in dict_two.keys():
list_cor = dict_two[p]
for i in list_cor:
# print i
csv_w.writerow(i)
f.close()
# write the dictionary to csv file
f = open(align_two_task_time_gap_csv, 'a+')
csv_w = csv.writer(f)
x = []
y1 = []
y2 = []
for p in dict_two.keys():
list_cor = dict_two[p]
for i in list_cor:
gap_1 = float(i[3]) - float(i[2])
gap_2 = float(i[5]) - float(i[4])
csv_w.writerow([i[0], i[1], gap_1, gap_2])
x.append(i[0] + '&' + i[1])
y1.append(gap_1)
y2.append(gap_2)
f.close()
# plot the diagram
viz = Visdom()
assert viz.check_connection()
# ----------------------------------------------------------------------------------------------
# plot all the points via matplotlib.pyplot
# ----------------------------------------------------------------------------------------------
try:
import matplotlib.pyplot as plt
plt.switch_backend('agg')
# plt.plot(y1[6500:7000])
plt.plot(y2)
plt.ylabel('numerical numbers')
win = viz.matplot(plt)
except BaseException as err:
print('Skipped matplotlib example')
print('Error message: ', err)
assert viz.win_exists(win), 'Created window marked as not existing'
class VanillaSeqToSeq(nn.Module):
def __init__(self,
embedding_size,
hidden_size,
max_len,
max_r,
lang,
path,
task,
lr=0.01,
n_layers=1,
dropout=0.1,
env_name=None,
debug_host_ip=None):
super(VanillaSeqToSeq, self).__init__()
# Check input parameters
assert env_name != None
assert debug_host_ip != None
if embedding_size == None:
warnings.warn(
'Word embedding size are none,set to be same with hiddensize:{}'
.format(hidden_size))
embedding_size = hidden_size
self.name = "VanillaSeqToSeq"
self.task = task
self.input_size = lang.n_words
self.output_size = lang.n_words
self.hidden_size = hidden_size
self.max_len = max_len ## max input
self.max_r = max_r ## max responce len
self.lang = lang
self.lr = lr
self.decoder_learning_ratio = 1.0
self.n_layers = n_layers
self.dropout = dropout
if path:
if USE_CUDA:
logging.info("MODEL {} LOADED".format(str(path)))
self.encoder = torch.load(str(path) + '/enc.th')
self.decoder = torch.load(str(path) + '/dec.th')
else:
logging.info("MODEL {} LOADED".format(str(path)))
self.encoder = torch.load(
str(path) + '/enc.th', lambda storage, loc: storage)
self.decoder = torch.load(
str(path) + '/dec.th', lambda storage, loc: storage)
self.decoder.viz_arr = []
else:
self.encoder = EncoderRNN(input_size=lang.n_words,
word_size=embedding_size,
hidden_size=hidden_size,
n_layers=n_layers,
dropout=dropout)
self.decoder = VanillaDecoderRNN(hidden_size=hidden_size,
word_size=embedding_size,
output_size=lang.n_words,
max_len=self.max_len,
n_layers=n_layers,
dropout=dropout)
# Initialize optimizers and criterion
self.encoder_optimizer = optim.Adam(self.encoder.parameters(), lr=lr)
self.decoder_optimizer = optim.Adam(self.decoder.parameters(),
lr=lr *
self.decoder_learning_ratio)
self.loss = 0
self.print_every = 1
self.print_loss_avg = 0
# Move models to GPU
if USE_CUDA:
self.encoder.cuda()
self.decoder.cuda()
try:
self.vis = Visdom(server=debug_host_ip, env=env_name)
print('server ip:{},env_name:{}'.format(debug_host_ip, env_name))
except:
print("Can't not use visdom ")
self.vis = None
def print_loss(self):
print_loss_avg = self.loss / self.print_every
self.print_loss_avg = print_loss_avg
self.print_every += 1
return 'L:{:.2f}'.format(print_loss_avg)
def save_model(self, dec_type):
name_data = "KVR/" if self.task == '' else "BABI/"
if USEKB:
directory = 'save/vanilla_KB-' + name_data + str(
self.task) + 'HDD' + str(self.hidden_size) + 'DR' + str(
self.dropout) + 'L' + str(self.n_layers) + 'lr' + str(
self.lr) + str(dec_type)
else:
directory = 'save/vanilla_noKB-' + name_data + str(
self.task) + 'HDD' + str(self.hidden_size) + 'DR' + str(
self.dropout) + 'L' + str(self.n_layers) + 'lr' + str(
self.lr) + str(dec_type)
if not os.path.exists(directory):
os.makedirs(directory)
torch.save(self.encoder, directory + '/enc.th')
torch.save(self.decoder, directory + '/dec.th')
def load_model(self, file_name_enc, file_name_dec):
self.encoder = torch.load(file_name_enc)
self.decoder = torch.load(file_name_dec)
def train_batch(self, input_batches, input_lengths, target_batches,
target_lengths, target_index, target_gate, batch_size,
clip, teacher_forcing_ratio, reset):
# Zero gradients of both optimizers
if reset:
self.loss = 0
self.print_every = 1
self.encoder_optimizer.zero_grad()
self.decoder_optimizer.zero_grad()
# self.opt.zero_grad()
loss_Vocab, loss_Ptr, loss_Gate = 0, 0, 0
# Run words through encoder
encoder_outputs, encoder_hidden = self.encoder(input_batches,
input_lengths)
# Prepare input and output variables
decoder_input = Variable(torch.LongTensor([SOS_token] * batch_size))
decoder_hidden = (encoder_hidden[0][:self.decoder.n_layers],
encoder_hidden[1][:self.decoder.n_layers])
max_target_length = max(target_lengths)
all_decoder_outputs_vocab = Variable(
torch.zeros(max_target_length, batch_size, self.output_size))
# Move new Variables to CUDA
if USE_CUDA:
all_decoder_outputs_vocab = all_decoder_outputs_vocab.cuda()
decoder_input = decoder_input.cuda()
# Choose whether to use teacher forcing
use_teacher_forcing = random.random() < teacher_forcing_ratio
if use_teacher_forcing:
# Run through decoder one time step at a time
for t in range(max_target_length):
decoder_vacab, decoder_hidden = self.decoder(
decoder_input, decoder_hidden, encoder_outputs)
all_decoder_outputs_vocab[t] = decoder_vacab
decoder_input = target_batches[
t] # Next input is current target
if USE_CUDA: decoder_input = decoder_input.cuda()
else:
for t in range(max_target_length):
decoder_vacab, decoder_hidden = self.decoder(
decoder_input, decoder_hidden, encoder_outputs)
all_decoder_outputs_vocab[t] = decoder_vacab
topv, topi = decoder_vacab.data.topk(1)
decoder_input = Variable(
topi.view(-1)) # Chosen word is next input
if USE_CUDA: decoder_input = decoder_input.cuda()
#Loss calculation and backpropagation
loss_Vocab = masked_cross_entropy(
all_decoder_outputs_vocab.transpose(
0, 1).contiguous(), # -> batch x seq
target_batches.transpose(0, 1).contiguous(), # -> batch x seq
target_lengths)
loss = loss_Vocab
loss.backward()
# Clip gradient norms
torch.nn.utils.clip_grad_norm_(self.encoder.parameters(), clip)
torch.nn.utils.clip_grad_norm_(self.decoder.parameters(), clip)
# Update parameters with optimizers
self.encoder_optimizer.step()
self.decoder_optimizer.step()
# self.opt.step()
self.loss += loss.item()
def evaluate_batch(self, batch_size, input_batches, input_lengths,
target_batches):
# Set to not-training mode to disable dropout
self.encoder.train(False)
self.decoder.train(False)
# Run words through encoder
encoder_outputs, encoder_hidden = self.encoder(input_batches,
input_lengths, None)
# Prepare input and output variables
decoder_input = Variable(torch.LongTensor([SOS_token] * batch_size))
decoder_hidden = (encoder_hidden[0][:self.decoder.n_layers],
encoder_hidden[1][:self.decoder.n_layers])
decoded_words = []
all_decoder_outputs_vocab = Variable(
torch.zeros(self.max_r, batch_size, self.decoder.output_size))
# Move new Variables to CUDA
if USE_CUDA:
all_decoder_outputs_vocab = all_decoder_outputs_vocab.cuda()
decoder_input = decoder_input.cuda()
# Run through decoder one time step at a time
for t in range(self.max_r):
decoder_vacab, decoder_hidden = self.decoder(
decoder_input, decoder_hidden, encoder_outputs)
all_decoder_outputs_vocab[t] = decoder_vacab
topv, topi = decoder_vacab.data.topk(1)
decoder_input = Variable(topi.view(-1))
decoded_words.append([
'<EOS>' if ni == EOS_token else self.lang.index2word[ni.item()]
for ni in topi.view(-1)
])
# Next input is chosen word
if USE_CUDA: decoder_input = decoder_input.cuda()
# Set back to training mode
self.encoder.train(True)
self.decoder.train(True)
return decoded_words
def evaluate(self,
dev,
avg_best,
epoch,
BLEU=False,
Analyse=False,
type='dev'):
assert type == 'dev' or type == 'test'
logging.info("STARTING EVALUATION:{}".format(type))
acc_avg = 0.0
wer_avg = 0.0
bleu_avg = 0.0
acc_P = 0.0
acc_V = 0.0
microF1_PRED,microF1_PRED_cal,microF1_PRED_nav,microF1_PRED_wet = [],[],[],[]
microF1_TRUE,microF1_TRUE_cal,microF1_TRUE_nav,microF1_TRUE_wet = [],[],[],[]
ref = []
hyp = []
ref_s = ""
hyp_s = ""
dialog_acc_dict = {}
pbar = tqdm(enumerate(dev), total=len(dev))
if Analyse == True:
write_fp = write_to_disk('./vanilla-seq-generate.txt')
for j, data_dev in pbar:
words = self.evaluate_batch(len(data_dev[1]), data_dev[0],
data_dev[1], data_dev[2])
acc = 0
w = 0
temp_gen = []
#print(words)
for i, row in enumerate(np.transpose(words)):
st = ''
for e in row:
if e == '<EOS>':
break
else:
st += e + ' '
temp_gen.append(st)
correct = data_dev[7][i]
### compute F1 SCORE
if args['dataset'] == 'kvr':
f1_true, f1_pred = computeF1(data_dev[8][i],
st.lstrip().rstrip(),
correct.lstrip().rstrip())
microF1_TRUE += f1_true
microF1_PRED += f1_pred
f1_true, f1_pred = computeF1(data_dev[9][i],
st.lstrip().rstrip(),
correct.lstrip().rstrip())
microF1_TRUE_cal += f1_true
microF1_PRED_cal += f1_pred
f1_true, f1_pred = computeF1(data_dev[10][i],
st.lstrip().rstrip(),
correct.lstrip().rstrip())
microF1_TRUE_nav += f1_true
microF1_PRED_nav += f1_pred
f1_true, f1_pred = computeF1(data_dev[11][i],
st.lstrip().rstrip(),
correct.lstrip().rstrip())
microF1_TRUE_wet += f1_true
microF1_PRED_wet += f1_pred
elif args['dataset'] == 'babi' and int(self.task) == 6:
f1_true, f1_pred = computeF1(data_dev[-2][i],
st.lstrip().rstrip(),
correct.lstrip().rstrip())
microF1_TRUE += f1_true
microF1_PRED += f1_pred
if args['dataset'] == 'babi':
if data_dev[-1][i] not in dialog_acc_dict.keys():
dialog_acc_dict[data_dev[-1][i]] = []
if (correct.lstrip().rstrip() == st.lstrip().rstrip()):
acc += 1
dialog_acc_dict[data_dev[-1][i]].append(1)
else:
dialog_acc_dict[data_dev[-1][i]].append(0)
else:
if (correct.lstrip().rstrip() == st.lstrip().rstrip()):
acc += 1
#else:
# print("Correct:"+str(correct.lstrip().rstrip()))
# print("\tPredict:"+str(st.lstrip().rstrip()))
# print("\tFrom:"+str(self.from_whichs[:,i]))
# w += wer(correct.lstrip().rstrip(),st.lstrip().rstrip())
ref.append(str(correct.lstrip().rstrip()))
hyp.append(str(st.lstrip().rstrip()))
ref_s += str(correct.lstrip().rstrip()) + "\n"
hyp_s += str(st.lstrip().rstrip()) + "\n"
# write batch data to disk
if Analyse == True:
for gen, gold in zip(temp_gen, data_dev[7]):
write_fp.write(gen + '\t' + gold + '\n')
acc_avg += acc / float(len(data_dev[1]))
wer_avg += w / float(len(data_dev[1]))
pbar.set_description("R:{:.4f},W:{:.4f}".format(
acc_avg / float(len(dev)), wer_avg / float(len(dev))))
if Analyse == True:
write_fp.close()
if args['dataset'] == 'babi':
# TODO:计算平均的对话准确度
dia_acc = 0
for k in dialog_acc_dict.keys():
if len(dialog_acc_dict[k]) == sum(dialog_acc_dict[k]):
dia_acc += 1
logging.info("Dialog Accuracy:\t" +
str(dia_acc * 1.0 / len(dialog_acc_dict.keys())))
self._send_metrics(epoch,
type,
acc=dia_acc * 1.0 / len(dialog_acc_dict.keys()))
self._send_metrics(epoch,
type,
acc_response=acc_avg / float(len(dev)))
if args['dataset'] == 'kvr':
f1 = f1_score(microF1_TRUE, microF1_PRED, average='micro')
f1_cal = f1_score(microF1_TRUE_cal,
microF1_PRED_cal,
average='micro')
f1_wet = f1_score(microF1_TRUE_wet,
microF1_PRED_wet,
average='micro')
f1_nav = f1_score(microF1_TRUE_nav,
microF1_PRED_nav,
average='micro')
logging.info("F1 SCORE:\t" + str(f1))
logging.info("F1 CAL:\t" + str(f1_cal))
logging.info("F1 WET:\t" + str(f1_wet))
logging.info("F1 NAV:\t" + str(f1_nav))
self._send_metrics(epoch,
type,
f1=f1,
f1_cal=f1_cal,
f1_wet=f1_wet,
f1_nav=f1_nav)
elif args['dataset'] == 'babi' and int(self.task) == 6:
f1 = f1_score(microF1_TRUE, microF1_PRED, average='micro')
logging.info("F1 SCORE:\t" + str(f1))
self._send_metrics(epoch, type, babi_6_f1=f1)
self._send_metrics(epoch, type, total_loss=self.print_loss_avg)
bleu_score = moses_multi_bleu(np.array(hyp),
np.array(ref),
lowercase=True)
logging.info("BLEU SCORE:" + str(bleu_score))
self._send_metrics(epoch, type, bleu=bleu_score)
if (BLEU):
if (bleu_score >= avg_best):
if type == 'dev':
self.save_model(str(self.name) + str(bleu_score))
logging.info("MODEL SAVED")
return bleu_score
else:
acc_avg = acc_avg / float(len(dev))
if (acc_avg >= avg_best):
if type == 'dev':
self.save_model(str(self.name) + str(acc_avg))
logging.info("MODEL SAVED")
return acc_avg
def _send_metrics(self, epoch, type, **krgs):
def show(win, epoch, value):
opts = {}
opts['title'] = win
if self.vis.win_exists(win):
self.vis.line(X=np.array([epoch]),
Y=np.array([value]),
win=win,
update='append',
opts=opts)
else:
self.vis.line(X=np.array([epoch]),
Y=np.array([value]),
win=win,
opts=opts)
if self.vis:
for key, value in krgs.items():
show(type + key, epoch, value)
opts=dict(
markersize=10,
markercolor=np.floor(np.random.random((2, 3)) * 255),
),
)
win = viz.scatter(
X=np.random.rand(255, 2),
opts=dict(
markersize=10,
markercolor=np.random.randint(0, 255, (255, 3,)),
),
)
# assert that the window exists
assert viz.win_exists(win)
# add new trace to scatter plot
viz.scatter(
X=np.random.rand(255),
Y=np.random.rand(255),
win=win,
name='new_trace',
update='new'
)
# bar plots
viz.bar(X=np.random.rand(20))
viz.bar(
X=np.abs(np.random.rand(5, 3)),
class VisdomGrapher:
def __init__(self, env_name, server, port=8097):
self.env_name = env_name
self.vis = Visdom(server=server, port=port, env=env_name)
startup_sec = 1
while not self.vis.check_connection() and startup_sec > 0:
time.sleep(0.1)
startup_sec -= 0.1
# optional, time out connection
def add_scalar(self, y, x, plot_name, idtag, opts={}):
'''
Update vidomplot by win_title with a scalar value.
If it doesn't exist, create a new plot
- win_title: name and title of plot
- y: y coord
- x: x coord
- options_dict, example {'legend': 'NAME', 'ytickmin': 0, 'ytickmax': 1}
'''
# todo:numpy check for y and x
# check if graph exists
exists = self.vis.win_exists(idtag)
# update existing window
if exists:
self.vis.line(Y=np.array([y]),
X=np.array([x]),
win=idtag,
update='append',
opts=opts)
else:
self.vis.line(Y=np.array([y]),
X=np.array([x]),
win=idtag,
opts={
'title': plot_name,
'xlabel': 'epoch'
})
def add_histogram(self, x, plot_name, idtag, opts={'numbins': 25}):
if len(list(x.shape)) > 1:
x = x.ravel()
opts = {**opts, **{'title': plot_name}}
self.vis.histogram(x, win=idtag, opts=opts)
def add_image(self, image, plot_name, idtag):
'''
Update visdomplot by win_title with a scalar value.
If it doesn't exist, create a new plot by default
'''
self.vis.images(image, win=idtag, opts=dict(title=plot_name))
def add_tensor_grid(self, batch_tensor, plot_name, idtag, nrow):
# .permute(1, 2, 0)
grid_image = torchvision.utils.make_grid(batch_tensor, nrow=nrow)
self.add_image(grid_image, plot_name, idtag)
def add_scatter2d(self, x, labels, plot_name, idtag, reinit=True, opts={}):
# check if graph exists
exists = self.vis.win_exists(idtag) and not reinit
# check for single values
if x.shape[0] == 1:
x = np.array([x])
labels = np.array([labels])
# Labels should start at 1
if np.amin(labels) == 0:
labels += 1
# update existing window
if exists:
self.vis.scatter(x, labels, win=idtag, update='append', opts=opts)
else:
self.vis.scatter(x, labels, win=idtag, opts={'title': plot_name})
default=DEFAULT_VISDOM_PORT,
help='IP port of the visdom server')
parser.add_argument('--value',
type=float,
default=DEFAULT_DATA_VALUE,
help='Y value for the line plot')
args = parser.parse_args()
print("Connecting to visdom server on ", args.visdom_host, ":",
args.visdom_port)
value = args.value
viz = Visdom(server="http://" + args.visdom_host, port=args.visdom_port)
assert viz.check_connection()
if not viz.win_exists(WIN_LINE):
viz.line(Y=np.array([1]),
X=np.array([1]),
opts=dict(
xlabel='Iteration',
ylabel='Rate',
title='Bitcoin to Chinese Yuan',
),
win=WIN_LINE)
if os.path.exists(store_file):
f = open(store_file, 'rb')
iteration = int(pickle.load(f))
f.close()
else:
iteration = 0
opts=dict(
markersize=10,
markercolor=np.floor(np.random.random((2, 3)) * 255),
),
)
win = viz.scatter(
X=np.random.rand(255, 2),
opts=dict(
markersize=10,
markercolor=np.random.randint(0, 255, (255, 3,)),
),
)
# assert that the window exists
assert viz.win_exists(win), 'Created window marked as not existing'
# add new trace to scatter plot
viz.scatter(
X=np.random.rand(255),
Y=np.random.rand(255),
win=win,
name='new_trace',
update='new'
)
# 2D scatter plot with text labels:
viz.scatter(
X=np.random.rand(10, 2),
opts=dict(
textlabels=['Label %d' % (i + 1) for i in range(10)]
),
)
win = viz.scatter(
X=np.random.rand(255, 2),
opts=dict(
markersize=10,
markercolor=np.random.randint(0, 255, (
255,
3,
)),
),
)
# assert that the window exists
assert viz.win_exists(win), 'Created window marked as not existing'
# add new trace to scatter plot
viz.scatter(X=np.random.rand(255),
Y=np.random.rand(255),
win=win,
name='new_trace',
update='new')
# 2D scatter plot with text labels:
viz.scatter(X=np.random.rand(10, 2),
opts=dict(textlabels=['Label %d' % (i + 1)
for i in range(10)]))
viz.scatter(X=np.random.rand(10, 2),
Y=[1] * 5 + [2] * 3 + [3] * 2,
opts=dict(legend=['A', 'B', 'C'],
