def __init__(self, server='localhost', port=8097): self.server = server self.port = port try: vis = Visdom(server="http://"+self.server, port=self.port) vis.win_exists("test")#dummy call to make sure that a connection to # visdom can be established except: print("\n\n\TortillaError :: Unable to connect to Visdom Server even when you have plotting on. \n\ Are you sure that you have visdom server running at : \ http://{}:{} ? \n \ If not, please start visdom by running : \n\npython -m visdom.server \n\n\n \ Or...disable plotting by passing the --no-plots option. \ ".format(server,port)) exit(0)
class BaseVis(object): def __init__(self, viz_opts, update_mode='append', env=None, win=None, resume=False, port=8097): self.viz_opts = viz_opts self.update_mode = update_mode self.win = win if env is None: env = 'main' self.viz = Visdom(env=env, port=port) # if resume first plot should not update with replace self.removed = not resume def win_exists(self): return self.viz.win_exists(self.win) def close(self): if self.win is not None: self.viz.close(win=self.win) self.win = None def register_event_handler(self, handler): self.viz.register_event_handler(handler, self.win)
def create_summary_window(vis: visdom.Visdom, visdom_env_name: str, experiment_name: str, summary: str) -> str: return vis.text( text=summary, win=experiment_name, env=visdom_env_name, opts={'title': 'Summary', 'width': 576, 'height': 416}, append=vis.win_exists(experiment_name, visdom_env_name) )
), ) win = viz.scatter( X=np.random.rand(255, 2), opts=dict( markersize=10, markercolor=np.random.randint(0, 255, ( 255, 3, )), ), ) # assert that the window exists assert viz.win_exists(win), 'Created window marked as not existing' # add new trace to scatter plot viz.scatter(X=np.random.rand(255), Y=np.random.rand(255), win=win, name='new_trace', update='new') # 2D scatter plot with text labels: viz.scatter(X=np.random.rand(10, 2), opts=dict(textlabels=['Label %d' % (i + 1) for i in range(10)])) viz.scatter(X=np.random.rand(10, 2), Y=[1] * 5 + [2] * 3 + [3] * 2, opts=dict(legend=['A', 'B', 'C'],
), ) win = viz.scatter( X=np.random.rand(255, 2), opts=dict( markersize=10, markercolor=np.random.randint(0, 255, ( 255, 3, )), ), ) # assert that the window exists assert viz.win_exists(win) # add new trace to scatter plot viz.updateTrace( X=np.random.rand(255), Y=np.random.rand(255), win=win, name='new_trace', ) # bar plots viz.bar(X=np.random.rand(20)) viz.bar(X=np.abs(np.random.rand(5, 3)), opts=dict(stacked=True, legend=['Facebook', 'Google', 'Twitter'], rownames=['2012', '2013', '2014', '2015', '2016']))
class Visualizer(object): def __init__(self, config: Config): # logging_level = logging._checkLevel("INFO") # logging.getLogger().setLevel(logging_level) # VisdomServer.start_server(port=VisdomServer.DEFAULT_PORT, env_path=config.vis_env_path) self.reinit(config) def reinit(self, config): self.config = config try: self.visdom = Visdom(env=config.visdom_env) self.connected = self.visdom.check_connection() if not self.connected: print( "Visdom server hasn't started, please run command 'python -m visdom.server' in terminal." ) # try: # print("Visdom server hasn't started, do you want to start it? ") # if 'y' in input("y/n: ").lower(): # os.popen('python -m visdom.server') # except Exception as e: # warn(e) except ConnectionError as e: warn("Can't open Visdom because " + e.strerror) with open(self.config.log_file, 'a') as f: info = "[{time}]Initialize Visdom\n".format( time=timestr('%m-%d %H:%M:%S')) info += str(self.config) f.write(info + '\n') def save(self, save_path: str = None) -> str: retstr = self.visdom.save([ self.config.visdom_env ]) # return current environments name in format of json try: ret = json.loads(retstr)[0] if ret == self.config.visdom_env: if isinstance(save_path, str): from shutil import copy copy(self.config.vis_env_path, save_path) print('Visdom Environment has saved into ' + save_path) else: print('Visdom Environment has saved into ' + self.config.vis_env_path) with open(self.config.vis_env_path, 'r') as fp: env_str = json.load(fp) return env_str except Exception as e: warn(e) return None def clear(self): self.visdom.close() @staticmethod def _to_numpy(value): if isinstance(value, t.Tensor): value = value.cpu().detach().numpy() elif isinstance(value, np.ndarray): pass else: value = np.array(value) if value.ndim == 0: value = value[np.newaxis] return value def plot(self, y, x, line_name, win, legend=None): # type:(float,float,str,str,list)->bool """Plot a (sequence) of y point(s) (each) with one x value(s), loop this method to draw whole plot""" update = None if not self.visdom.win_exists(win) else 'append' opts = dict(title=win) if legend is not None: opts["legend"] = legend y = Visualizer._to_numpy(y) x = Visualizer._to_numpy(x) return win == self.visdom.line(y, x, win=win, env=self.config.visdom_env, update=update, name=line_name, opts=opts) def bar(self, y, win, rowindices=None): opts = dict(title=win) y = Visualizer._to_numpy(y) if isinstance(rowindices, list) and len(rowindices) == len(y): opts["rownames"] = rowindices return win == self.visdom.bar(y, win=win, env=self.config.visdom_env, opts=opts) def log(self, msg, name, append=True, log_file=None): # type:(Visualizer,str,str,bool,str)->bool if log_file is None: log_file = self.config.log_file info = "[{time}]{msg}".format(time=timestr('%m-%d %H:%M:%S'), msg=msg) append = append and self.visdom.win_exists(name) ret = self.visdom.text(info, win=name, env=self.config.visdom_env, opts=dict(title=name), append=append) mode = 'a+' if append else 'w+' with open(log_file, mode) as f: f.write(info + '\n') return ret == name def log_process(self, num, total, msg, name, append=True): # type:(Visualizer,int,int,str,str,bool)->bool info = "[{time}]{msg}".format(time=timestr('%m-%d %H:%M:%S'), msg=msg) append = append and self.visdom.win_exists(name) ret = self.visdom.text(info, win=(name), env=self.config.visdom_env, opts=dict(title=name), append=append) with open(self.config.log_file, 'a') as f: f.write(info + '\n') self.process_bar(num, total, msg) return ret == name def process_bar(self, num, total, msg='', length=50): rate = num / total rate_num = int(rate * 100) clth = int(rate * length) if len(msg) > 0: msg += ':' # msg = msg.replace('\n', '').replace('\r', '') if rate_num == 100: r = '\r%s[%s%d%%]\n' % ( msg, '*' * length, rate_num, ) else: r = '\r%s[%s%s%d%%]' % ( msg, '*' * clth, '-' * (length - clth), rate_num, ) sys.stdout.write(r) sys.stdout.flush() return r.replace('\r', ':')
def plot_line(vis: visdom.Visdom, window_name: str, env: Optional[str] = None, line_label: Optional[str] = None, x: Optional[np.ndarray] = None, y: Optional[np.ndarray] = None, x_label: Optional[str] = None, y_label: Optional[str] = None, width: int = 576, height: int = 416, draw_marker: bool = False) -> str: empty_call = not vis.win_exists(window_name) if empty_call and (x is not None or y is not None): return window_name if x is None: x = np.ones(1) empty_call = empty_call & True if y is None: y = np.full(1, np.nan) empty_call = empty_call & True if x.shape != y.shape: x = np.ones_like(y) opts = { 'showlegend': True, 'markers': draw_marker, 'markersize': 5, } if empty_call: opts['title'] = window_name opts['width'] = width opts['height'] = height window_name = vis.line( X=x, Y=y, win=window_name, env=env, update='append', name=line_label, opts=opts ) xtickmin, xtickmax = 0.0, np.max(x) * 1.05 ytickmin, ytickmax = calc_ytick_range(vis, window_name, env) opts = { 'showlegend': True, 'xtickmin': xtickmin, 'xtickmax': xtickmax, 'ytickmin': ytickmin, 'ytickmax': ytickmax, 'xlabel': x_label, 'ylabel': y_label } window_name = vis.update_window_opts(win=window_name, opts=opts, env=env) return window_name
def build_logger(build_state, checkpoint={}, run='NavA3C', port=8097): vis = Visdom(port=port) env = run wins = mp.Manager().dict() offset = checkpoint.setdefault('offset', -1) + 1 if 'plots' in checkpoint: wins = dict((name, id) for name, id in checkpoint['plots'].items() if vis.win_exists(id, env)) def _save_checkpoint(step): if step % args.save_interval != 0 or args.checkpoint_path is None: return checkpoint_path = os.path.abspath(args.checkpoint_path) checkpoint_dir = os.path.dirname(checkpoint_path) if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) state = build_state() state['plots'] = dict(wins) state['offset'] = offset torch.save(state, args.checkpoint_path) def _log_scatter(value, step, win_name, title, mode='train'): if mode == 'test': step += offset elif step % args.log_interval != 0: return if not vis.check_connection(): return if isinstance(value, torch.Tensor): norm = value.numpy() else: norm = value win_id = wins.setdefault(win_name) if win_id is None: wins[win_name] = vis.scatter(X=np.array([[step, norm]]), win=win_id, env=env, opts=dict(title=title)) else: vis.scatter(X=np.array([[step, norm]]), win=win_id, env=env, update='append') vis.save([env]) def _log_reward(total_reward, step, mode): if mode == 'train' and step % args.log_interval != 0: return if not vis.check_connection(): return win_name = 'total_reward_{}'.format(mode) win_id = wins.setdefault(win_name) if win_id is None: wins[win_name] = vis.line( np.array([0, 0]), win=win_id, env=env, opts=dict(title='{} reward'.format(mode))) else: vis.line(Y=np.array([total_reward]), X=np.array([step]), win=win_id, env=env, update='append') vis.save([env]) def _log_video(video, step): step += offset if args.video_path is None: return video_path = os.path.abspath(args.video_path) video_dir = os.path.dirname(video_path) if not os.path.exists(video_dir): os.makedirs(video_dir) skvideo.io.vwrite(video_path, np.array(video)) if not vis.check_connection(): return win_name = 'last_test_episode' win_id = wins.setdefault(win_name) if win_id is None: wins[win_name] = vis.video( videofile=video_path, win=win_id, env=env, opts=dict(title='episode {}'.format(step))) else: vis.video(videofile=video_path, win=win_id, env=env, opts=dict(title='episode {}'.format(step))) vis.save([env]) return dict( video=_log_video, grad_norm=lambda n, s: _log_scatter(n, s, 'grad_norm', 'gradient norm' ), train_reward=lambda r, s: _log_reward(r, s, 'train'), test_reward=lambda r, s: _log_reward(r, s, 'test'), train_time=lambda n, s: _log_scatter( n, s, 'train_time', 'training wall time (per episode)', 'train'), test_time=lambda n, s: _log_scatter( n, s, 'test_time', 'evaluation wall time (per episode)', 'test'), checkpoint=_save_checkpoint)
def align_execute_mean_two(): temp = [] f = open(two_task_time_csv, 'r') csv_r = csv.reader(f) for p in csv_r: temp.append(p) f.close() length = len(temp) print('length of rough pearson correlation list: %d' % length) # Build the dictionary for storing the complemented correlation str_i = '-' inference_exe_join = [] for i in inference_exe: # inference_exe_join.append(str_i.join(i)) inference_exe_join.append(i[2] + '_' + i[3] + '_' + i[1]) dict_two = {} # em_exe = poly_exe + inference_exe_join em_exe = poly_exe + inference_exe_join + example_exe for p in em_exe: dict_two[p] = [] # search each kind of benchmark and complement all the combinations with it for i in range(length): obj = temp[i] print('*** %s *** & *** %s ***' % (obj[0], obj[1])) group = temp[i] if group[0] != group[1]: x1, x2 = dict_two[group[0]], dict_two[group[1]] x1.append(group) x2.append( [group[1], group[0], group[4], group[5], group[2], group[3]]) dict_two[group[0]], dict_two[group[1]] = x1, x2 else: x1 = dict_two[group[0]] x1.append(group) dict_two[group[0]] = x1 # write the dictionary to csv file f = open(align_two_task_time_csv, 'a+') csv_w = csv.writer(f) for p in dict_two.keys(): list_cor = dict_two[p] for i in list_cor: # print i csv_w.writerow(i) f.close() # write the dictionary to csv file f = open(align_two_task_time_gap_csv, 'a+') csv_w = csv.writer(f) x = [] y1 = [] y2 = [] for p in dict_two.keys(): list_cor = dict_two[p] for i in list_cor: gap_1 = float(i[3]) - float(i[2]) gap_2 = float(i[5]) - float(i[4]) csv_w.writerow([i[0], i[1], gap_1, gap_2]) x.append(i[0] + '&' + i[1]) y1.append(gap_1) y2.append(gap_2) f.close() # plot the diagram viz = Visdom() assert viz.check_connection() # ---------------------------------------------------------------------------------------------- # plot all the points via matplotlib.pyplot # ---------------------------------------------------------------------------------------------- try: import matplotlib.pyplot as plt plt.switch_backend('agg') # plt.plot(y1[6500:7000]) plt.plot(y2) plt.ylabel('numerical numbers') win = viz.matplot(plt) except BaseException as err: print('Skipped matplotlib example') print('Error message: ', err) assert viz.win_exists(win), 'Created window marked as not existing'
class VanillaSeqToSeq(nn.Module): def __init__(self, embedding_size, hidden_size, max_len, max_r, lang, path, task, lr=0.01, n_layers=1, dropout=0.1, env_name=None, debug_host_ip=None): super(VanillaSeqToSeq, self).__init__() # Check input parameters assert env_name != None assert debug_host_ip != None if embedding_size == None: warnings.warn( 'Word embedding size are none,set to be same with hiddensize:{}' .format(hidden_size)) embedding_size = hidden_size self.name = "VanillaSeqToSeq" self.task = task self.input_size = lang.n_words self.output_size = lang.n_words self.hidden_size = hidden_size self.max_len = max_len ## max input self.max_r = max_r ## max responce len self.lang = lang self.lr = lr self.decoder_learning_ratio = 1.0 self.n_layers = n_layers self.dropout = dropout if path: if USE_CUDA: logging.info("MODEL {} LOADED".format(str(path))) self.encoder = torch.load(str(path) + '/enc.th') self.decoder = torch.load(str(path) + '/dec.th') else: logging.info("MODEL {} LOADED".format(str(path))) self.encoder = torch.load( str(path) + '/enc.th', lambda storage, loc: storage) self.decoder = torch.load( str(path) + '/dec.th', lambda storage, loc: storage) self.decoder.viz_arr = [] else: self.encoder = EncoderRNN(input_size=lang.n_words, word_size=embedding_size, hidden_size=hidden_size, n_layers=n_layers, dropout=dropout) self.decoder = VanillaDecoderRNN(hidden_size=hidden_size, word_size=embedding_size, output_size=lang.n_words, max_len=self.max_len, n_layers=n_layers, dropout=dropout) # Initialize optimizers and criterion self.encoder_optimizer = optim.Adam(self.encoder.parameters(), lr=lr) self.decoder_optimizer = optim.Adam(self.decoder.parameters(), lr=lr * self.decoder_learning_ratio) self.loss = 0 self.print_every = 1 self.print_loss_avg = 0 # Move models to GPU if USE_CUDA: self.encoder.cuda() self.decoder.cuda() try: self.vis = Visdom(server=debug_host_ip, env=env_name) print('server ip:{},env_name:{}'.format(debug_host_ip, env_name)) except: print("Can't not use visdom ") self.vis = None def print_loss(self): print_loss_avg = self.loss / self.print_every self.print_loss_avg = print_loss_avg self.print_every += 1 return 'L:{:.2f}'.format(print_loss_avg) def save_model(self, dec_type): name_data = "KVR/" if self.task == '' else "BABI/" if USEKB: directory = 'save/vanilla_KB-' + name_data + str( self.task) + 'HDD' + str(self.hidden_size) + 'DR' + str( self.dropout) + 'L' + str(self.n_layers) + 'lr' + str( self.lr) + str(dec_type) else: directory = 'save/vanilla_noKB-' + name_data + str( self.task) + 'HDD' + str(self.hidden_size) + 'DR' + str( self.dropout) + 'L' + str(self.n_layers) + 'lr' + str( self.lr) + str(dec_type) if not os.path.exists(directory): os.makedirs(directory) torch.save(self.encoder, directory + '/enc.th') torch.save(self.decoder, directory + '/dec.th') def load_model(self, file_name_enc, file_name_dec): self.encoder = torch.load(file_name_enc) self.decoder = torch.load(file_name_dec) def train_batch(self, input_batches, input_lengths, target_batches, target_lengths, target_index, target_gate, batch_size, clip, teacher_forcing_ratio, reset): # Zero gradients of both optimizers if reset: self.loss = 0 self.print_every = 1 self.encoder_optimizer.zero_grad() self.decoder_optimizer.zero_grad() # self.opt.zero_grad() loss_Vocab, loss_Ptr, loss_Gate = 0, 0, 0 # Run words through encoder encoder_outputs, encoder_hidden = self.encoder(input_batches, input_lengths) # Prepare input and output variables decoder_input = Variable(torch.LongTensor([SOS_token] * batch_size)) decoder_hidden = (encoder_hidden[0][:self.decoder.n_layers], encoder_hidden[1][:self.decoder.n_layers]) max_target_length = max(target_lengths) all_decoder_outputs_vocab = Variable( torch.zeros(max_target_length, batch_size, self.output_size)) # Move new Variables to CUDA if USE_CUDA: all_decoder_outputs_vocab = all_decoder_outputs_vocab.cuda() decoder_input = decoder_input.cuda() # Choose whether to use teacher forcing use_teacher_forcing = random.random() < teacher_forcing_ratio if use_teacher_forcing: # Run through decoder one time step at a time for t in range(max_target_length): decoder_vacab, decoder_hidden = self.decoder( decoder_input, decoder_hidden, encoder_outputs) all_decoder_outputs_vocab[t] = decoder_vacab decoder_input = target_batches[ t] # Next input is current target if USE_CUDA: decoder_input = decoder_input.cuda() else: for t in range(max_target_length): decoder_vacab, decoder_hidden = self.decoder( decoder_input, decoder_hidden, encoder_outputs) all_decoder_outputs_vocab[t] = decoder_vacab topv, topi = decoder_vacab.data.topk(1) decoder_input = Variable( topi.view(-1)) # Chosen word is next input if USE_CUDA: decoder_input = decoder_input.cuda() #Loss calculation and backpropagation loss_Vocab = masked_cross_entropy( all_decoder_outputs_vocab.transpose( 0, 1).contiguous(), # -> batch x seq target_batches.transpose(0, 1).contiguous(), # -> batch x seq target_lengths) loss = loss_Vocab loss.backward() # Clip gradient norms torch.nn.utils.clip_grad_norm_(self.encoder.parameters(), clip) torch.nn.utils.clip_grad_norm_(self.decoder.parameters(), clip) # Update parameters with optimizers self.encoder_optimizer.step() self.decoder_optimizer.step() # self.opt.step() self.loss += loss.item() def evaluate_batch(self, batch_size, input_batches, input_lengths, target_batches): # Set to not-training mode to disable dropout self.encoder.train(False) self.decoder.train(False) # Run words through encoder encoder_outputs, encoder_hidden = self.encoder(input_batches, input_lengths, None) # Prepare input and output variables decoder_input = Variable(torch.LongTensor([SOS_token] * batch_size)) decoder_hidden = (encoder_hidden[0][:self.decoder.n_layers], encoder_hidden[1][:self.decoder.n_layers]) decoded_words = [] all_decoder_outputs_vocab = Variable( torch.zeros(self.max_r, batch_size, self.decoder.output_size)) # Move new Variables to CUDA if USE_CUDA: all_decoder_outputs_vocab = all_decoder_outputs_vocab.cuda() decoder_input = decoder_input.cuda() # Run through decoder one time step at a time for t in range(self.max_r): decoder_vacab, decoder_hidden = self.decoder( decoder_input, decoder_hidden, encoder_outputs) all_decoder_outputs_vocab[t] = decoder_vacab topv, topi = decoder_vacab.data.topk(1) decoder_input = Variable(topi.view(-1)) decoded_words.append([ '<EOS>' if ni == EOS_token else self.lang.index2word[ni.item()] for ni in topi.view(-1) ]) # Next input is chosen word if USE_CUDA: decoder_input = decoder_input.cuda() # Set back to training mode self.encoder.train(True) self.decoder.train(True) return decoded_words def evaluate(self, dev, avg_best, epoch, BLEU=False, Analyse=False, type='dev'): assert type == 'dev' or type == 'test' logging.info("STARTING EVALUATION:{}".format(type)) acc_avg = 0.0 wer_avg = 0.0 bleu_avg = 0.0 acc_P = 0.0 acc_V = 0.0 microF1_PRED,microF1_PRED_cal,microF1_PRED_nav,microF1_PRED_wet = [],[],[],[] microF1_TRUE,microF1_TRUE_cal,microF1_TRUE_nav,microF1_TRUE_wet = [],[],[],[] ref = [] hyp = [] ref_s = "" hyp_s = "" dialog_acc_dict = {} pbar = tqdm(enumerate(dev), total=len(dev)) if Analyse == True: write_fp = write_to_disk('./vanilla-seq-generate.txt') for j, data_dev in pbar: words = self.evaluate_batch(len(data_dev[1]), data_dev[0], data_dev[1], data_dev[2]) acc = 0 w = 0 temp_gen = [] #print(words) for i, row in enumerate(np.transpose(words)): st = '' for e in row: if e == '<EOS>': break else: st += e + ' ' temp_gen.append(st) correct = data_dev[7][i] ### compute F1 SCORE if args['dataset'] == 'kvr': f1_true, f1_pred = computeF1(data_dev[8][i], st.lstrip().rstrip(), correct.lstrip().rstrip()) microF1_TRUE += f1_true microF1_PRED += f1_pred f1_true, f1_pred = computeF1(data_dev[9][i], st.lstrip().rstrip(), correct.lstrip().rstrip()) microF1_TRUE_cal += f1_true microF1_PRED_cal += f1_pred f1_true, f1_pred = computeF1(data_dev[10][i], st.lstrip().rstrip(), correct.lstrip().rstrip()) microF1_TRUE_nav += f1_true microF1_PRED_nav += f1_pred f1_true, f1_pred = computeF1(data_dev[11][i], st.lstrip().rstrip(), correct.lstrip().rstrip()) microF1_TRUE_wet += f1_true microF1_PRED_wet += f1_pred elif args['dataset'] == 'babi' and int(self.task) == 6: f1_true, f1_pred = computeF1(data_dev[-2][i], st.lstrip().rstrip(), correct.lstrip().rstrip()) microF1_TRUE += f1_true microF1_PRED += f1_pred if args['dataset'] == 'babi': if data_dev[-1][i] not in dialog_acc_dict.keys(): dialog_acc_dict[data_dev[-1][i]] = [] if (correct.lstrip().rstrip() == st.lstrip().rstrip()): acc += 1 dialog_acc_dict[data_dev[-1][i]].append(1) else: dialog_acc_dict[data_dev[-1][i]].append(0) else: if (correct.lstrip().rstrip() == st.lstrip().rstrip()): acc += 1 #else: # print("Correct:"+str(correct.lstrip().rstrip())) # print("\tPredict:"+str(st.lstrip().rstrip())) # print("\tFrom:"+str(self.from_whichs[:,i])) # w += wer(correct.lstrip().rstrip(),st.lstrip().rstrip()) ref.append(str(correct.lstrip().rstrip())) hyp.append(str(st.lstrip().rstrip())) ref_s += str(correct.lstrip().rstrip()) + "\n" hyp_s += str(st.lstrip().rstrip()) + "\n" # write batch data to disk if Analyse == True: for gen, gold in zip(temp_gen, data_dev[7]): write_fp.write(gen + '\t' + gold + '\n') acc_avg += acc / float(len(data_dev[1])) wer_avg += w / float(len(data_dev[1])) pbar.set_description("R:{:.4f},W:{:.4f}".format( acc_avg / float(len(dev)), wer_avg / float(len(dev)))) if Analyse == True: write_fp.close() if args['dataset'] == 'babi': # TODO:计算平均的对话准确度 dia_acc = 0 for k in dialog_acc_dict.keys(): if len(dialog_acc_dict[k]) == sum(dialog_acc_dict[k]): dia_acc += 1 logging.info("Dialog Accuracy:\t" + str(dia_acc * 1.0 / len(dialog_acc_dict.keys()))) self._send_metrics(epoch, type, acc=dia_acc * 1.0 / len(dialog_acc_dict.keys())) self._send_metrics(epoch, type, acc_response=acc_avg / float(len(dev))) if args['dataset'] == 'kvr': f1 = f1_score(microF1_TRUE, microF1_PRED, average='micro') f1_cal = f1_score(microF1_TRUE_cal, microF1_PRED_cal, average='micro') f1_wet = f1_score(microF1_TRUE_wet, microF1_PRED_wet, average='micro') f1_nav = f1_score(microF1_TRUE_nav, microF1_PRED_nav, average='micro') logging.info("F1 SCORE:\t" + str(f1)) logging.info("F1 CAL:\t" + str(f1_cal)) logging.info("F1 WET:\t" + str(f1_wet)) logging.info("F1 NAV:\t" + str(f1_nav)) self._send_metrics(epoch, type, f1=f1, f1_cal=f1_cal, f1_wet=f1_wet, f1_nav=f1_nav) elif args['dataset'] == 'babi' and int(self.task) == 6: f1 = f1_score(microF1_TRUE, microF1_PRED, average='micro') logging.info("F1 SCORE:\t" + str(f1)) self._send_metrics(epoch, type, babi_6_f1=f1) self._send_metrics(epoch, type, total_loss=self.print_loss_avg) bleu_score = moses_multi_bleu(np.array(hyp), np.array(ref), lowercase=True) logging.info("BLEU SCORE:" + str(bleu_score)) self._send_metrics(epoch, type, bleu=bleu_score) if (BLEU): if (bleu_score >= avg_best): if type == 'dev': self.save_model(str(self.name) + str(bleu_score)) logging.info("MODEL SAVED") return bleu_score else: acc_avg = acc_avg / float(len(dev)) if (acc_avg >= avg_best): if type == 'dev': self.save_model(str(self.name) + str(acc_avg)) logging.info("MODEL SAVED") return acc_avg def _send_metrics(self, epoch, type, **krgs): def show(win, epoch, value): opts = {} opts['title'] = win if self.vis.win_exists(win): self.vis.line(X=np.array([epoch]), Y=np.array([value]), win=win, update='append', opts=opts) else: self.vis.line(X=np.array([epoch]), Y=np.array([value]), win=win, opts=opts) if self.vis: for key, value in krgs.items(): show(type + key, epoch, value)
opts=dict( markersize=10, markercolor=np.floor(np.random.random((2, 3)) * 255), ), ) win = viz.scatter( X=np.random.rand(255, 2), opts=dict( markersize=10, markercolor=np.random.randint(0, 255, (255, 3,)), ), ) # assert that the window exists assert viz.win_exists(win) # add new trace to scatter plot viz.scatter( X=np.random.rand(255), Y=np.random.rand(255), win=win, name='new_trace', update='new' ) # bar plots viz.bar(X=np.random.rand(20)) viz.bar( X=np.abs(np.random.rand(5, 3)),
class VisdomGrapher: def __init__(self, env_name, server, port=8097): self.env_name = env_name self.vis = Visdom(server=server, port=port, env=env_name) startup_sec = 1 while not self.vis.check_connection() and startup_sec > 0: time.sleep(0.1) startup_sec -= 0.1 # optional, time out connection def add_scalar(self, y, x, plot_name, idtag, opts={}): ''' Update vidomplot by win_title with a scalar value. If it doesn't exist, create a new plot - win_title: name and title of plot - y: y coord - x: x coord - options_dict, example {'legend': 'NAME', 'ytickmin': 0, 'ytickmax': 1} ''' # todo:numpy check for y and x # check if graph exists exists = self.vis.win_exists(idtag) # update existing window if exists: self.vis.line(Y=np.array([y]), X=np.array([x]), win=idtag, update='append', opts=opts) else: self.vis.line(Y=np.array([y]), X=np.array([x]), win=idtag, opts={ 'title': plot_name, 'xlabel': 'epoch' }) def add_histogram(self, x, plot_name, idtag, opts={'numbins': 25}): if len(list(x.shape)) > 1: x = x.ravel() opts = {**opts, **{'title': plot_name}} self.vis.histogram(x, win=idtag, opts=opts) def add_image(self, image, plot_name, idtag): ''' Update visdomplot by win_title with a scalar value. If it doesn't exist, create a new plot by default ''' self.vis.images(image, win=idtag, opts=dict(title=plot_name)) def add_tensor_grid(self, batch_tensor, plot_name, idtag, nrow): # .permute(1, 2, 0) grid_image = torchvision.utils.make_grid(batch_tensor, nrow=nrow) self.add_image(grid_image, plot_name, idtag) def add_scatter2d(self, x, labels, plot_name, idtag, reinit=True, opts={}): # check if graph exists exists = self.vis.win_exists(idtag) and not reinit # check for single values if x.shape[0] == 1: x = np.array([x]) labels = np.array([labels]) # Labels should start at 1 if np.amin(labels) == 0: labels += 1 # update existing window if exists: self.vis.scatter(x, labels, win=idtag, update='append', opts=opts) else: self.vis.scatter(x, labels, win=idtag, opts={'title': plot_name})
default=DEFAULT_VISDOM_PORT, help='IP port of the visdom server') parser.add_argument('--value', type=float, default=DEFAULT_DATA_VALUE, help='Y value for the line plot') args = parser.parse_args() print("Connecting to visdom server on ", args.visdom_host, ":", args.visdom_port) value = args.value viz = Visdom(server="http://" + args.visdom_host, port=args.visdom_port) assert viz.check_connection() if not viz.win_exists(WIN_LINE): viz.line(Y=np.array([1]), X=np.array([1]), opts=dict( xlabel='Iteration', ylabel='Rate', title='Bitcoin to Chinese Yuan', ), win=WIN_LINE) if os.path.exists(store_file): f = open(store_file, 'rb') iteration = int(pickle.load(f)) f.close() else: iteration = 0
opts=dict( markersize=10, markercolor=np.floor(np.random.random((2, 3)) * 255), ), ) win = viz.scatter( X=np.random.rand(255, 2), opts=dict( markersize=10, markercolor=np.random.randint(0, 255, (255, 3,)), ), ) # assert that the window exists assert viz.win_exists(win), 'Created window marked as not existing' # add new trace to scatter plot viz.scatter( X=np.random.rand(255), Y=np.random.rand(255), win=win, name='new_trace', update='new' ) # 2D scatter plot with text labels: viz.scatter( X=np.random.rand(10, 2), opts=dict( textlabels=['Label %d' % (i + 1) for i in range(10)]