def visdom_server(): # Start Visdom server once and stop it with visdom_server_stop global vd_hostname, vd_port, vd_server_process if vd_server_process is None: import subprocess import time from visdom import Visdom from visdom.server import download_scripts download_scripts() vd_hostname = "localhost" vd_port = random.randint(8089, 8887) try: vis = Visdom(server=vd_hostname, port=vd_port, raise_exceptions=True) except ConnectionError: pass vd_server_process = subprocess.Popen( ["python", "-m", "visdom.server", "--hostname", vd_hostname, "-port", str(vd_port)] ) time.sleep(5) vis = Visdom(server=vd_hostname, port=vd_port) assert vis.check_connection() vis.close() yield (vd_hostname, vd_port)
class BaseVis(object): def __init__(self, viz_opts, update_mode='append', env=None, win=None, resume=False, port=8097): self.viz_opts = viz_opts self.update_mode = update_mode self.win = win if env is None: env = 'main' self.viz = Visdom(env=env, port=port) # if resume first plot should not update with replace self.removed = not resume def win_exists(self): return self.viz.win_exists(self.win) def close(self): if self.win is not None: self.viz.close(win=self.win) self.win = None def register_event_handler(self, handler): self.viz.register_event_handler(handler, self.win)
class ZcsVisdom: def __init__(self, server='10.160.82.54', port=8899): self.vis = Visdom(server=server, port=port) assert self.vis.check_connection() self.wins = {} def plot(self, data, win): ''' array: np.array win: str ''' x = np.arange(data.shape[0]) self.vis.line(data, x, win=win)#, update='append') def append(self, data, win, opts): ''' data是个list,长度和绘制的折线数一致 ''' assert isinstance(data, list) if win not in self.wins.keys(): self.vis.close(win) self.wins[win] = 0 y = [data] x = [self.wins[win]] self.vis.line(Y=y, X=x, win=win, update='append', opts=opts) self.wins[win] += 1
class VisdomLinePlotter(object): """Plots to Visdom""" def __init__(self, env_name='main'): self.viz = Visdom() self.env = env_name self.plots = {} def clear(self): self.viz.close(None, env=self.env) def plot(self, var_name, split_name, title_name, x, y): if var_name not in self.plots: self.plots[var_name] = self.viz.line(X=np.array([x, x]), Y=np.array([y, y]), env=self.env, opts=dict(legend=[split_name], title=title_name, xlabel='Epochs', ylabel=var_name)) else: self.viz.line(X=np.array([x]), Y=np.array([y]), env=self.env, win=self.plots[var_name], name=split_name, update='append')
class VisdomLinePlotter(object): """Plots to Visdom""" def __init__(self, env_name='main', port=8097): self.viz = Visdom(port=port) self.env = env_name self.plots = {} self.scores_window = None self.image_window = None def plot(self, var_name, split_name, x, y, x_label='Epochs'): if var_name not in self.plots: self.plots[var_name] = self.viz.line(X=np.array([x,x]), Y=np.array([y,y]), env=self.env, opts=dict( legend=[split_name], title=var_name, xlabel=x_label, ylabel=var_name )) else: self.viz.line(X=np.array([x]), Y=np.array([y]), env=self.env, win=self.plots[var_name], name=split_name, update = 'append') def close_window(self, var_name): self.viz.close(self.plots[var_name]) del self.plots[var_name] def images(self, images): if self.image_window != None: self.viz.close(self.image_window) self.image_window = self.viz.images(images, nrow=3, env=self.env, opts=dict(nrow=2, title='Images Batch'))
class VisdomLinePlotter(object): """Plots to Visdom""" def __init__(self, env_name='main'): self.viz = Visdom(port=8090, env=env_name) self.env = env_name self.plots = {} self.scores_window = None self.image_window = None def plot(self, var_name, split_name, x, y, x_label='Epochs'): if var_name not in self.plots: self.plots[var_name] = self.viz.line(X=np.array([x, x]), Y=np.array([y, y]), env=self.env, opts=dict(legend=[split_name], title=var_name, xlabel=x_label, ylabel=var_name)) else: self.viz.line(X=np.array([x]), Y=np.array([y]), env=self.env, win=self.plots[var_name], name=split_name, update='append') def close_window(self, var_name): self.viz.close(self.plots[var_name]) del self.plots[var_name] def plot_voxels(self, name, voxels, title, savePLY=False): v, f = cubes2mesh(name, voxels, save=savePLY) self.viz.mesh(X=v, Y=f, opts=dict(opacity=0.5, title=title))
class VisdomLogger: def __init__(self, visdom_env='main', log_every=10, prefix=''): self.vis = None self.log_every = log_every self.prefix = prefix if visdom_env is not None: self.vis = Visdom(env=visdom_env) self.vis.close() def on_batch_end(self, state): iters = state['iters'] if self.log_every != -1 and iters % self.log_every == 0: self.log(iters, state['metrics']) def on_epoch_end(self, state): self.log(state['iters'], state['metrics']) def log(self, iters, xs, store_history=[]): if self.vis is None: return for name, x in xs.items(): name = self.prefix + name if isinstance(x, (float, int)): self.vis.line(X=[iters], Y=[x], update='append', win=name, opts=dict(title=name), name=name) elif isinstance(x, str): self.vis.text(x, win=name, opts=dict(title=name)) elif isinstance(x, torch.Tensor): if x.numel() == 1: self.vis.line(X=[iters], Y=[x.item()], update='append', win=name, opts=dict(title=name), name=name) elif x.dim() == 2: self.vis.heatmap(x, win=name, opts=dict(title=name)) elif x.dim() == 3: self.vis.image(x, win=name, opts=dict(title=name, store_history=name in store_history)) elif x.dim() == 4: self.vis.images(x, win=name, opts=dict(title=name, store_history=name in store_history)) else: assert False, "incorrect tensor dim" else: assert False, "incorrect type " + x.__class__.__name__
def visdom_plot(total_num_steps, cum_reward): from visdom import Visdom global vis global win1 global win2 global avg_reward if vis is None: vis = Visdom() assert vis.check_connection() # Close all existing plots vis.close() # Running average for curve smoothing # avg_reward = avg_reward * 0.9 + 0.1 * cum_reward avg_reward = cum_reward / total_num_steps X.append(total_num_steps) Y1.append(cum_reward) Y2.append(avg_reward) # The plot with the handle 'win' is updated each time this is called win1 = vis.line( X=np.array(X), Y=np.array(Y1), opts=dict( #title = 'All Environments', xlabel='Total number of steps', ylabel='Cumulative reward', ytickmin=0, #ytickmax=1, #ytickstep=0.1, #legend=legend, #showlegend=True, width=900, height=500), win=win1) # The plot with the handle 'win' is updated each time this is called win2 = vis.line( X=np.array(X), Y=np.array(Y2), opts=dict( # title = 'All Environments', xlabel='Total number of episodes', ylabel='Average Reward', ytickmin=0, # ytickmax=1, # ytickstep=0.1, # legend=legend, # showlegend=True, width=900, height=500), win=win2)
def testCallbackUpdateGraph(self): with subprocess.Popen(['python', '-m', 'visdom.server', '-port', str(self.port)]) as proc: # wait for visdom server startup (any better way?) viz = Visdom(server=self.host, port=self.port) for attempt in range(5): time.sleep(1.0) # seconds if viz.check_connection(): break assert viz.check_connection() viz.close() self.model.update(self.corpus) proc.kill()
class VisdomLogger(): """ Logger that uses visdom to create learning curves Parameters ---------- - env: str, name of the visdom environment - log_checkpoints: bool, whether to use checkpoints or epoch averages for training loss - legend: tuple, names of the different losses that will be plotted. """ def __init__(self, server='http://localhost', port=8097): if Visdom is None: warnings.warn("Couldn't import visdom: `pip install visdom`") else: self.viz = Visdom(server=server, port=port) # self.viz.delete_env() def deleteWindow(self, win): self.viz.close(win=win) def appendLine(self, name, win, X, Y, xlabel='empty', ylabel='empty'): if xlabel == 'empty' or ylabel == 'empty': self.viz.line(X=X, Y=Y, win=win, name=name, update='append', opts=dict(title="Loss")) else: self.viz.line(X=X, Y=Y, win=win, name=name, update='append', opts=dict(title="Loss", xlabel=xlabel, ylabel=ylabel, showlegend=True)) def plotLine(self, name, win, X, Y): self.viz.line(X=X, Y=Y, win=win, name=name) def plotImage(self, image, win, title="Image", caption="Just a Image"): self.viz.image(image, win=win, opts=dict(title=title, caption=caption)) def plotImages(self, images, win, nrow, caption="Validation Output"): self.viz.images(images, win=win, nrow=nrow, opts=dict(caption=caption)) def plot3dScatter(self, point, win): print("Point is", point) self.viz.scatter(X = point, win=win, opts=dict(update='update'))
def create_session(**kwargs): """ Creates a visdom session Parameters ---------- kwargs : ... Returns ------- object a visdom session """ session = Visdom(**kwargs) session.close(None) return session
class Visualizer: def __init__(self, env="main"): self._viz = Visdom(env=env, use_incoming_socket=False) self._viz.close(env=env) def plot_line(self, values, steps, name, legend=None): if legend is None: opts = dict(title=name) else: opts = dict(title=name, legend=legend) self._viz.line(X=numpy.column_stack(steps), Y=numpy.column_stack(values), win=name, update='append', opts=opts) def plot_text(self, text, title, pre=True): _width = max([len(x) for x in text.split("\n")]) * 10 _heigth = len(text.split("\n")) * 20 _heigth = max(_heigth, 120) if pre: text = "<pre>{}</pre>".format(text) self._viz.text(text, win=title, opts=dict(title=title, width=min(_width, 400), height=min(_heigth, 400))) def plot_scatter(self, data, labels, title): X = numpy.concatenate(data, axis=0) Y = numpy.concatenate( [numpy.full(len(d), i) for i, d in enumerate(data, 1)], axis=0) self._viz.scatter(win=title, X=X, Y=Y, opts=dict(legend=labels, title=title, markersize=5, webgl=True, width=600, height=600, markeropacity=0.5))
def testCallbackUpdateGraph(self): # Popen have no context-manager in 2.7, for this reason - try/finally. try: # spawn visdom.server proc = subprocess.Popen(['python', '-m', 'visdom.server', '-port', str(self.port)]) # wait for visdom server startup (any better way?) time.sleep(3) viz = Visdom(server=self.host, port=self.port) assert viz.check_connection() # clear screen viz.close() self.model.update(self.corpus) finally: proc.kill()
class VisdomSummary(object): def __init__(self, port=None, env=None): self.vis = Visdom(port=port, env=env) self.opts = dict() def scalar(self, win, name, x, y, remove=False): if not hasattr(self, '__scalar'): self.__scalar = Scalar() opts = dict(title=win) self.__scalar.update(self.vis, win, name, x, y, opts=opts, remove=remove) def bar(self, win, x, rownames=None): if rownames is None: rownames = ['{}'.format(i) for i in range(x.size(0))] opts = dict(title=win, rownames=rownames) self.vis.bar(X=x, win=win, opts=opts) def image2d(self, win, name, img, caption=None, nrow=3): if not hasattr(self, '__image2d'): self.__image2d = Image2D() self.__image2d.update(self.vis, win, name, img, caption, nrow) def image3d(self, win, name, img): raise NotImplementedError def text(self, win, text): self.vis.text(text, win=win) def close(self, win=None): self.vis.close(win=win) def save(self): self.vis.save([self.vis.env])
def testCallbackUpdateGraph(self): # Popen have no context-manager in 2.7, for this reason - try/finally. try: # spawn visdom.server proc = subprocess.Popen( ['python', '-m', 'visdom.server', '-port', str(self.port)]) # wait for visdom server startup (any better way?) time.sleep(3) viz = Visdom(server=self.host, port=self.port) assert viz.check_connection() # clear screen viz.close() self.model.update(self.corpus) finally: proc.kill()
def visdom_plot(total_num_steps, mean_reward): # Lazily import visdom so that people don't need to install visdom # if they're not actually using it from visdom import Visdom global vis global win global avg_reward if vis is None: vis = Visdom() assert vis.check_connection() # Close all existing plots vis.close() # Running average for curve smoothing avg_reward = avg_reward * 0.9 + 0.1 * mean_reward X.append(total_num_steps) Y.append(avg_reward) # The plot with the handle 'win' is updated each time this is called win = vis.line( X=np.array(X), Y=np.array(Y), opts=dict( #title = 'All Environments', xlabel='Total time steps', ylabel='Reward per episode', ytickmin=0, #ytickmax=1, #ytickstep=0.1, #legend=legend, #showlegend=True, width=900, height=500), win=win)
def testCallbackUpdateGraph(self): # create a new process for visdom.server proc = subprocess.Popen(['python', '-m', 'visdom.server']) # wait for visdom server startup time.sleep(3) # create visdom object print(FLAGS.port) print(FLAGS.server) viz = Visdom(port=FLAGS.port, server=FLAGS.server) # check connection assert viz.check_connection() # clear screen viz.close() # test callback's update graph function try: self.model.update(self.corpus) # raise AttributeError("test") except Exception as e: print(e) # kill visdom.server proc.kill() print('server killed') self.assertTrue(0) # kill visdom.server try: proc.wait(timeout=3) except subprocess.TimeoutExpired: proc.kill() print('server killed')
k = [0, 7, 2, 3, 6, 7, 1, 1, 5, 5, 7, 6] Y = np.c_[i, j, k] viz.mesh(X=X, Y=Y, opts=dict(opacity=0.5)) # SVG plotting svgstr = """ <svg height="300" width="300"> <ellipse cx="80" cy="80" rx="50" ry="30" style="fill:red;stroke:purple;stroke-width:2" /> Sorry, your browser does not support inline SVG. </svg> """ viz.svg(svgstr=svgstr, opts=dict(title='Example of SVG Rendering')) # close text window: viz.close(win=textwindow) # assert that the closed window doesn't exist assert not viz.win_exists(textwindow), 'Closed window still exists' # Arbitrary visdom content trace = dict(x=[1, 2, 3], y=[4, 5, 6], mode="markers+lines", type='custom', marker={ 'color': 'red', 'symbol': 104, 'size': "10" }, text=["one", "two", "three"],
for target, actual in zip(target_output, output): target_str = embedded_to_string(target) actual_str = embedded_to_string(actual) print(target_str + ": " + actual_str) if target_str == actual_str: num_correct += 1 print("Accuracy: %f" % (num_correct / output.shape[0])) last_100_losses = [] if summarize and rnn.debug: loss = np.mean(last_save_losses) last_save_losses = [] print(random_length) if (epoch / summarize_freq) % 3 == 0: viz.close(None) viz.heatmap( mhx['memory'][0], opts=dict(xtickstep=10, ytickstep=2, title='Memory, t: ' + str(epoch) + ', num: ' + q_and_a_to_string(input_data[0], output[0]), ylabel='layer * time', xlabel='mem_slot * mem_size')) all_read_weights = torch.stack( [x['read_weights'][0] for x in all_mems]) viz.heatmap( all_read_weights.squeeze(), opts=dict(xtickstep=10, ytickstep=2,
### TORCH ### import torch use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if use_cuda else "cpu") Tensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor print("Using GPU:" + str(use_cuda)) ### VISDOM ### from visdom import Visdom viz = Visdom(port=8097) viz.close() optsLossTrain = { 'title': 'Loss Train', 'xlabel': 'Epoch', 'width': 800, 'height': 400, } winLossTrain = viz.line(Y=Tensor([0]), X=Tensor([0]), opts=optsLossTrain) optsLossTest = { 'title': 'Loss Test', 'xlabel': 'Epoch', 'width': 800, 'height': 400,
z = [0, 0, 0, 0, 1, 1, 1, 1] X = np.c_[x, y, z] i = [7, 0, 0, 0, 4, 4, 6, 6, 4, 0, 3, 2] j = [3, 4, 1, 2, 5, 6, 5, 2, 0, 1, 6, 3] k = [0, 7, 2, 3, 6, 7, 1, 1, 5, 5, 7, 6] Y = np.c_[i, j, k] viz.mesh(X=X, Y=Y, opts=dict(opacity=0.5)) # SVG plotting svgstr = """ <svg height="300" width="300"> <ellipse cx="80" cy="80" rx="50" ry="30" style="fill:red;stroke:purple;stroke-width:2" /> Sorry, your browser does not support inline SVG. </svg> """ viz.svg( svgstr=svgstr, opts=dict(title='Example of SVG Rendering') ) # close text window: viz.close(win=textwindow) # PyTorch tensor try: import torch viz.line(Y=torch.Tensor([[0., 0.], [1., 1.]])) except ImportError: print('Skipped PyTorch example')
class VisdomReporter(Reporter): def __init__(self, port=6006, save_dir=None): from visdom import Visdom super(VisdomReporter, self).__init__(save_dir) self._viz = Visdom(port=port, env=self._now) self._lines = defaultdict() assert self._viz.check_connection(), f""" Please launch visdom.server before calling VisdomReporter. $python -m visdom.server -port {port} """ def add_scalar(self, x, name: str, idx: int, **kwargs): self.add_scalars({name: x}, name=name, idx=idx, **kwargs) def add_scalars(self, x: dict, name, idx: int, **kwargs): x = {k: self._to_numpy(v) for k, v in x.items()} num_lines = len(x) is_new = self._lines.get(name) is None self._lines[name] = 1 for k, v in x.items(): self._register_data(v, k, idx) opts = dict(title=name, legend=list(x.keys())) opts.update(**kwargs) X = np.column_stack((self._to_numpy(idx) for _ in range(num_lines))) Y = np.column_stack(x.values()) self._viz.line(X=X, Y=Y, update=None if is_new else "append", win=name, opts=opts) def add_parameters(self, x, name: str, idx: int, **kwargs): # todo raise NotImplementedError def add_text(self, x, name: str, idx: int): self._register_data(x, name, idx) self._viz.text(x) def add_image(self, x, name: str, idx: int): x, dim = self._tensor_type_check(x) assert dim == 3 self._viz.image(self._normalize(x), opts=dict(title="name", caption=str(idx))) def add_images(self, x, name: str, idx: int): x, dim = self._tensor_type_check(x) assert dim == 4 self._viz.images(self._normalize(x), opts=dict(title="name", caption=str(idx))) def _to_numpy(self, x): if isinstance(x, numbers.Number): x = np.array([x]) elif "Tensor" in str(type(x)): x = x.numpy() return def __exit__(self, exc_type, exc_val, exc_tb): super(VisdomReporter, self).__exit__(exc_type, exc_val, exc_tb) self._viz.close() @staticmethod def _normalize(x): # normalize tensor values in (0, 1) _min, _max = x.min(), x.max() return (x - _min) / (_max - _min)
class Visualizer: def __init__(self, env="main", server="http://localhost", port=8097, base_url="/", http_proxy_host=None, http_proxy_port=None): self._viz = Visdom(env=env, server=server, port=port, http_proxy_host=http_proxy_host, http_proxy_port=http_proxy_port, use_incoming_socket=False) self._viz.close(env=env) def plot_line(self, values, steps, name, legend=None): if legend is None: opts = dict(title=name) else: opts = dict(title=name, legend=legend) self._viz.line(X=numpy.column_stack(steps), Y=numpy.column_stack(values), win=name, update='append', opts=opts) def plot_text(self, text, title, pre=True): _width = max([len(x) for x in text.split("\n")]) * 10 _heigth = len(text.split("\n")) * 20 _heigth = max(_heigth, 120) if pre: text = "<pre>{}</pre>".format(text) self._viz.text(text, win=title, opts=dict(title=title, width=min(_width, 400), height=min(_heigth, 400))) def plot_bar(self, data, labels, title): self._viz.bar(win=title, X=data, opts=dict(legend=labels, stacked=False, title=title)) def plot_scatter(self, data, labels, title): X = numpy.concatenate(data, axis=0) Y = numpy.concatenate( [numpy.full(len(d), i) for i, d in enumerate(data, 1)], axis=0) self._viz.scatter(win=title, X=X, Y=Y, opts=dict(legend=labels, title=title, markersize=5, webgl=True, width=400, height=400, markeropacity=0.5)) def plot_heatmap(self, data, labels, title): self._viz.heatmap( win=title, X=data, opts=dict( title=title, columnnames=labels[1], rownames=labels[0], width=700, height=700, layoutopts={ 'plotly': { 'xaxis': { 'side': 'top', 'tickangle': -60, # 'autorange': "reversed" }, 'yaxis': { 'autorange': "reversed" }, } }))
class Custom_Visdom: def __init__(self, model_name, transfer_learning): self.vis = Visdom() if transfer_learning: self.vis.env = f'{model_name}_tl' else: self.vis.env = model_name self.vis.close() self.mod = sys.modules[__name__] self.pred_error = {} def loss_plot(self, checkpoint): opts = dict(title='Loss curve', legend=['train loss', 'valid_loss'], showlegend=True) length = 50 try: if len(checkpoint.epoch_list) < length: self._update_loss_plot(checkpoint.epoch_list, checkpoint.train_loss_list_per_epoch, checkpoint.valid_loss_list, opts, self.loss_plt) else: self._update_loss_plot( checkpoint.epoch_list[-length:], checkpoint.train_loss_list_per_epoch[-length:], checkpoint.valid_loss_list[-length:], opts, self.loss_plt) except: self.loss_plt = self.vis.line( X=np.array([checkpoint.epoch_list, checkpoint.epoch_list]).T, Y=np.array([ checkpoint.train_loss_list_per_epoch, checkpoint.valid_loss_list ]).T, opts=opts) def predict_plot(self, y_df, target='pre'): if target not in ['pre', 'trans']: raise ValueError('Wrong target') opts = dict(legend=y_df.columns.tolist(), showlegend=True) for y_target in y_df.columns: if y_target != 'pred': mse = mse_AIFrenz(y_df[y_target], y_df['pred']) self.pred_error[y_target] = mse if target == 'pre': opts['title'] = 'pretrain set prediction' try: self._update_predict_curve(y_df, opts, self.pretrain_pred) except: self.pretrain_pred = self.vis.line(X=np.tile( y_df.index, (y_df.shape[1], 1)).T, Y=y_df.values, opts=opts) else: opts['title'] = 'Y18 prediction' try: self._update_predict_curve(y_df, opts, self.trans_pred) except: self.trans_pred = self.vis.line(X=np.tile( y_df.index, (y_df.shape[1], 1)).T, Y=y_df.values, opts=opts) def print_error(self): text = '<h4> Error for each target </h4><br>' for keys, values in self.pred_error.items(): text += f'{keys}: {values:7.3f} <br>' try: self.vis.text(text, win=self.error, append=False) except: self.error = self.vis.text(text) def print_params(self, params): text = '<h4> Hyperparameter list </h4><br>' for keys, values in params.items(): text += f'{keys}: {values} <br>' self.vis.text(text) def print_training(self, EPOCH, epoch, training_time, avg_train_loss, valid_loss, patience, counter): iter_time = time.time() - training_time text = '<h4> Training status </h4><br>'\ f'\r Epoch: {epoch:3d}/{str(EPOCH):3s}<br>'\ f'train time: {int(iter_time//60):2d}m {iter_time%60:5.2f}s<br>'\ f'avg train loss: {avg_train_loss:7.3f}<br>'\ f'valid loss: {valid_loss:7.3f}<br>'\ f'\r EarlyStopping: {">"*counter + "-"*(patience-counter)} |<br>'\ # f'{"-----"*17}<br>' try: self.vis.text(text, win=self.training, append=False) except: self.training = self.vis.text(text) def _update_predict_curve(self, y_df, opts, win): self.vis.line(X=np.tile(y_df.index, (y_df.shape[1], 1)).T, Y=y_df.values, opts=opts, win=win, update='replace') def _update_loss_plot(self, epoch_list, train_loss_list_per_epoch, valid_loss_list, opts, win): self.vis.line(X=np.array([epoch_list, epoch_list]).T, Y=np.array([train_loss_list_per_epoch, valid_loss_list]).T, opts=opts, win=win, update='replace')
class Visualizer(object): def __init__(self, config: Config): # logging_level = logging._checkLevel("INFO") # logging.getLogger().setLevel(logging_level) # VisdomServer.start_server(port=VisdomServer.DEFAULT_PORT, env_path=config.vis_env_path) self.reinit(config) def reinit(self, config): self.config = config try: self.visdom = Visdom(env=config.visdom_env) self.connected = self.visdom.check_connection() if not self.connected: print( "Visdom server hasn't started, please run command 'python -m visdom.server' in terminal." ) # try: # print("Visdom server hasn't started, do you want to start it? ") # if 'y' in input("y/n: ").lower(): # os.popen('python -m visdom.server') # except Exception as e: # warn(e) except ConnectionError as e: warn("Can't open Visdom because " + e.strerror) with open(self.config.log_file, 'a') as f: info = "[{time}]Initialize Visdom\n".format( time=timestr('%m-%d %H:%M:%S')) info += str(self.config) f.write(info + '\n') def save(self, save_path: str = None) -> str: retstr = self.visdom.save([ self.config.visdom_env ]) # return current environments name in format of json try: ret = json.loads(retstr)[0] if ret == self.config.visdom_env: if isinstance(save_path, str): from shutil import copy copy(self.config.vis_env_path, save_path) print('Visdom Environment has saved into ' + save_path) else: print('Visdom Environment has saved into ' + self.config.vis_env_path) with open(self.config.vis_env_path, 'r') as fp: env_str = json.load(fp) return env_str except Exception as e: warn(e) return None def clear(self): self.visdom.close() @staticmethod def _to_numpy(value): if isinstance(value, t.Tensor): value = value.cpu().detach().numpy() elif isinstance(value, np.ndarray): pass else: value = np.array(value) if value.ndim == 0: value = value[np.newaxis] return value def plot(self, y, x, line_name, win, legend=None): # type:(float,float,str,str,list)->bool """Plot a (sequence) of y point(s) (each) with one x value(s), loop this method to draw whole plot""" update = None if not self.visdom.win_exists(win) else 'append' opts = dict(title=win) if legend is not None: opts["legend"] = legend y = Visualizer._to_numpy(y) x = Visualizer._to_numpy(x) return win == self.visdom.line(y, x, win=win, env=self.config.visdom_env, update=update, name=line_name, opts=opts) def bar(self, y, win, rowindices=None): opts = dict(title=win) y = Visualizer._to_numpy(y) if isinstance(rowindices, list) and len(rowindices) == len(y): opts["rownames"] = rowindices return win == self.visdom.bar(y, win=win, env=self.config.visdom_env, opts=opts) def log(self, msg, name, append=True, log_file=None): # type:(Visualizer,str,str,bool,str)->bool if log_file is None: log_file = self.config.log_file info = "[{time}]{msg}".format(time=timestr('%m-%d %H:%M:%S'), msg=msg) append = append and self.visdom.win_exists(name) ret = self.visdom.text(info, win=name, env=self.config.visdom_env, opts=dict(title=name), append=append) mode = 'a+' if append else 'w+' with open(log_file, mode) as f: f.write(info + '\n') return ret == name def log_process(self, num, total, msg, name, append=True): # type:(Visualizer,int,int,str,str,bool)->bool info = "[{time}]{msg}".format(time=timestr('%m-%d %H:%M:%S'), msg=msg) append = append and self.visdom.win_exists(name) ret = self.visdom.text(info, win=(name), env=self.config.visdom_env, opts=dict(title=name), append=append) with open(self.config.log_file, 'a') as f: f.write(info + '\n') self.process_bar(num, total, msg) return ret == name def process_bar(self, num, total, msg='', length=50): rate = num / total rate_num = int(rate * 100) clth = int(rate * length) if len(msg) > 0: msg += ':' # msg = msg.replace('\n', '').replace('\r', '') if rate_num == 100: r = '\r%s[%s%d%%]\n' % ( msg, '*' * length, rate_num, ) else: r = '\r%s[%s%s%d%%]' % ( msg, '*' * clth, '-' * (length - clth), rate_num, ) sys.stdout.write(r) sys.stdout.flush() return r.replace('\r', ':')
def main(): parser = argparse.ArgumentParser() parser.add_argument('--dataPath', type=str, default='data') parser.add_argument('--batchSz', type=int, default=64) parser.add_argument('--nEpochs', type=int, default=300) parser.add_argument('--num_workers', default=8, type=int, help='Number of workers used in dataloading') parser.add_argument('--no_cuda', type=str2bool, default=False) parser.add_argument('--procName', type=str, default='train', help='process name') parser.add_argument('--save', type=str, default='work/densenet.base', help='Location to save checkpoint models') parser.add_argument('--resume', '-r', type=str, default=None, help='resume from checkpoint') parser.add_argument('--seed', type=int, default=1) parser.add_argument( '--use_visdom', action='store_false', help='Whether or not to use visdom.the default is true') parser.add_argument('--viz_ip', type=str, default='http://localhost', help='server ip for visdom') parser.add_argument('--viz_port', type=int, default=8098, help='server port for visdom') parser.add_argument('--opt', type=str, default='sgd', choices=('sgd', 'adam', 'rmsprop')) args = parser.parse_args() args.cuda = not args.no_cuda and torch.cuda.is_available() classes = args.classes torch.manual_seed(args.seed) if args.cuda and torch.cuda.is_available(): torch.cuda.manual_seed(args.seed) setproctitle.setproctitle(args.procName) os.makedirs(args.save, exist_ok=True) if args.use_visdom: try: viz = Visdom(server=args.viz_ip, port=args.viz_port) assert viz.check_connection() viz.close() vis_title = args.procName epoch_plot = create_vis_plot(viz, vis_title) except BaseException as err: raise BaseException('Visdom connect error...') normMean = [0.49139968, 0.48215827, 0.44653124] normStd = [0.24703233, 0.24348505, 0.26158768] normTransform = transforms.Normalize(normMean, normStd) trainTransform = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normTransform ]) testTransform = transforms.Compose([transforms.ToTensor(), normTransform]) kwargs = {'pin_memory': True} if args.cuda else {} # trainLoader = DataLoader( # dset.CIFAR10(root='cifar', train=True, download=True, # transform=trainTransform), # batch_size=args.batchSz, shuffle=True, **kwargs) # testLoader = DataLoader( # dset.CIFAR10(root='cifar', train=False, download=True, # transform=testTransform), # batch_size=args.batchSz, shuffle=False, **kwargs) trainLoader = DataLoader(myImageFolder(root=ospj(args.dataPath, 'train'), label=ospj(args.dataPath, '../train.list'), transform=trainTransform), batch_size=args.batchSz, shuffle=True, num_workers=args.num_workers, **kwargs) testLoader = DataLoader(myImageFolder(root=ospj(args.dataPath, 'test'), label=ospj(args.dataPath, '../test.list'), transform=testTransform), batch_size=args.batchSz, shuffle=False, num_workers=args.num_workers, **kwargs) start_epoch = 0 if args.resume: print('===>Resuming from checkpoint:{} ..'.format(args.resume)) assert os.path.isfile( args.resume), 'Error:no checkpoint:%s found' % args.resume checkpoint = torch.load(args.resume) net = checkpoint['net'] start_epoch = checkpoint['epoch'] print('===>Resume from checkpoint:{},start epoch:{} ..'.format( args.resume, start_epoch)) else: net = densenet.DenseNet(growthRate=12, depth=100, reduction=0.5, bottleneck=True, nClasses=10) # use load_state_dict to reload parameters of model # if args.resume: # print('===>Resuming from checkpoint:{} ..'.format(args.resume)) # net = densenet.DenseNet(growthRate=12,depth=100,reduction=0.5,bottleneck=True,nClasses=10) # checkpoint = torch.load(args.resume, map_location=lambda storage, loc: storage) # start_epoch = checkpoint['epoch'] # net.load_state_dict(checkpoint['state_dict']) # else: # net = densenet.DenseNet(growthRate=12, depth=100, reduction=0.5, # bottleneck=True, nClasses=10) print(' + Number of params: {}'.format( sum([p.data.nelement() for p in net.parameters()]))) if args.cuda and torch.cuda.is_available(): net = net.cuda() # pdb.set_trace() if len(os.getenv('CUDA_VISIBLE_DEVICES')) > 1: net = torch.nn.DataParallel(net) cudnn.benchmark = True if args.opt == 'sgd': optimizer = optim.SGD(net.parameters(), lr=1e-1, momentum=0.9, weight_decay=1e-4) elif args.opt == 'adam': optimizer = optim.Adam(net.parameters(), weight_decay=1e-4) elif args.opt == 'rmsprop': optimizer = optim.RMSprop(net.parameters(), weight_decay=1e-4) # trainF = open(os.path.join(args.save, 'train.csv'), 'w') # testF = open(os.path.join(args.save, 'test.csv'), 'w') for epoch in range(start_epoch, args.nEpochs + 1): adjust_opt(args.opt, optimizer, epoch) train(args, epoch, net, trainLoader, optimizer, epoch_plot, viz) test(args, epoch, net, testLoader, optimizer, epoch_plot, viz) state = { 'net': net.module if len(os.getenv("CUDA_VISIBLE_DEVICES")) > 1 else net, # 'state_dict': net.module.state_dict() if len(os.getenv("CUDA_VISIBLE_DEVICES")) > 1 else net.state_dict(), 'epoch': epoch } torch.save(state, os.path.join(args.save, 'latest.pth'))
self.m.loss = totalvalue(rewards) * processlogprob(probs) self.m.optimize() if (i % plotinterval == 0): viz.line( X=list(range(len(actions))), Y=actions, opts=dict(legend=["a_mu", "a_sigma", "w_mu", "w_sigma"])) viz.line(X=list(range(len(states))), Y=states, opts=dict(legend=["v", "x", "y", "h", "d"])) def testcar(): car = Car() car.x = -5 car.y = -5 car.v = 1 car.w = 0.5 xs = [] ys = [] for t in range(100): car.step(t * 0.01) xs.append(car.x) ys.append(car.y) viz.quiver(X=xs, Y=ys) viz.close(win=None) sim = Simulation() sim.run(10001, 1000)
class NetFramework(): def __init__(self, defaults_path): parser = argparse.ArgumentParser( description='Net framework arguments description') parser.add_argument('--experiment', nargs='?', type=str, default='experiment', help='Experiment name') parser.add_argument('--model', nargs='?', type=str, default='', help='Architecture to use') parser.add_argument('--modelparam', type=str, default='{}', help='Experiment model parameters') parser.add_argument('--dataset', nargs='?', type=str, default='', help='Dataset key specified in dataconfig_*.json') parser.add_argument('--datasetparam', type=str, default='{}', help='Experiment dataset parameters') parser.add_argument('--imsize', nargs='?', type=int, default=200, help='Image resize parameter') parser.add_argument('--visdom', action='store_true', help='If included shows visdom visulaization') parser.add_argument( '--show_rate', nargs='?', type=int, default=4, help='Visdom show after num of iterations (used with --visdom)') parser.add_argument('--print_rate', nargs='?', type=int, default=4, help='Print after num of iterations') parser.add_argument( '--save_rate', nargs='?', type=int, default=10, help= 'Save after num of iterations (if --save_rate=0 then no save is done during training)' ) parser.add_argument('--use_cuda', nargs='?', type=int, default=0, help='GPU device (if --use_cuda=-1 then CPU used)') parser.add_argument( '--parallel', action='store_true', help='Use multiples GPU (used only if --use_cuda>-1)') parser.add_argument('--epochs', nargs='?', type=int, default=1000, help='Number of epochs') parser.add_argument('--batch_size', nargs='?', type=int, default=1, help='Minibatch size') parser.add_argument('--batch_acc', nargs='?', type=int, default=1, help='Minibatch accumulation') parser.add_argument('--train_worker', nargs='?', type=int, default=1, help='Number of training workers') parser.add_argument('--test_worker', nargs='?', type=int, default=1, help='Number of testing workers') parser.add_argument('--optimizer', nargs='?', type=str, default='RMSprop', help='Optimizer to use') parser.add_argument('--optimizerparam', type=str, default='{}', help='Experiment optimizer parameters') parser.add_argument('--lrschedule', nargs='?', type=str, default='none', help='LR Schedule to use') parser.add_argument('--loss', nargs='?', type=str, default='', help='Loss function to use') parser.add_argument('--lossparam', type=str, default='{}', help='Loss function parameters') parser.add_argument('--resume', action='store_true', help='Resume training') parser.add_argument('--seed', nargs='?', type=int, default=123, help='Random seed (for reproducibility)') args = parser.parse_args() if args.seed != -1: torch.manual_seed(args.seed) np.random.seed(args.seed) random.seed(args.seed) # create outputs folders root = '../out' experimentpath = (os.path.join(root, args.experiment)) folders = { 'root_path': root, 'experiment_path': experimentpath, 'model_path': os.path.join(experimentpath, 'model'), 'images_path': os.path.join(experimentpath, 'images') } for i in range(2): for folder, path in folders.items(): if not os.path.isdir(path): try: os.mkdir(path) except: pass json.dump(vars(args), open(os.path.join(experimentpath, 'args.json'), 'w')) args.folders = folders args.lossparam = json.loads(args.lossparam.replace("'", "\""), cls=Decoder) args.datasetparam = json.loads(args.datasetparam.replace("'", "\""), cls=Decoder) args.modelparam = json.loads(args.modelparam.replace("'", "\""), cls=Decoder) args.optimizerparam = json.loads(args.optimizerparam.replace( "'", "\""), cls=Decoder) # Parse use cuda self.device, self.use_parallel = parse_cuda(args) torch.cuda.set_device(args.use_cuda) # Visdom visualization self.visdom = args.visdom if self.visdom == True: self.vis = Visdom(use_incoming_socket=False) self.vis.close(env=args.experiment) self.visplotter = gph.VisdomLinePlotter(self.vis, env_name=args.experiment) self.visheatmap = gph.HeatMapVisdom(self.vis, env_name=args.experiment) self.visimshow = gph.ImageVisdom(self.vis, env_name=args.experiment) self.vistext = gph.TextVisdom(self.vis, env_name=args.experiment) # Showing results rate self.print_rate = args.print_rate self.show_rate = args.show_rate self.save_rate = args.save_rate self.init_epoch = 0 self.current_epoch = 0 self.epochs = args.epochs self.folders = args.folders self.bestmetric = 0 self.batch_size = args.batch_size self.batch_acc = args.batch_acc # Load datasets print('Loading dataset: ', args.dataset) self.traindataset, self.train_loader, self.dmodule = loaddataset( datasetname=args.dataset, experimentparam=args.datasetparam, batch_size=args.batch_size, worker=args.train_worker, config_file=os.path.join(defaults_path, 'dataconfig_train.json')) self.testdataset, self.test_loader, _ = loaddataset( datasetname=args.dataset, experimentparam=args.datasetparam, batch_size=args.batch_size, worker=args.test_worker, config_file=os.path.join(defaults_path, 'dataconfig_test.json')) self.warp_var_mod = import_module(self.dmodule + '.dataset') # Setup model print('Loading model: ', args.model) self.net, self.arch, self.mmodule = loadmodel( modelname=args.model, experimentparams=args.modelparam, config_file=os.path.join(defaults_path, 'modelconfig.json')) self.net.to(self.device) if self.use_parallel: self.net = torch.nn.DataParallel(self.net, device_ids=range( torch.cuda.device_count())) cudnn.benchmark = True # Setup Optimizer print('Selecting optimizer: ', args.optimizer) self.optimizer = selectoptimizer(args.optimizer, self.net, args.optimizerparam) # Setup Learning Rate Scheduling print('LR Schedule: ', args.lrschedule) self.scheduler = selectschedule(args.lrschedule, self.optimizer) # Setup Loss criterion print('Selecting loss function: ', args.loss) self.criterion, self.losseval = selectloss(lossname=args.loss, parameter=args.lossparam, config_file=os.path.join( defaults_path, 'loss_definition.json')) self.criterion.to(self.device) self.trlossavg = AverageMeter() self.vdlossavg = AverageMeter() # Others evaluation metrics print('Selecting metrics functions:') metrics_dict = get_metric_path( os.path.join(defaults_path, 'metrics.json')) self.metrics = dict() self.metrics_eval = dict() self.trmetrics_avg = dict() self.vdmetrics_avg = dict() for key, value in metrics_dict.items(): self.metrics[key], self.metrics_eval[key] = selectloss( lossname=value['metric'], parameter=value['param'], config_file=os.path.join(defaults_path, 'loss_definition.json')) self.metrics[key].to(self.device) self.trmetrics_avg[key] = AverageMeter() self.vdmetrics_avg[key] = AverageMeter() if args.resume: self.resume() signal.signal(signal.SIGTERM, self.savemodel) self.args = args def do_train(self): for current_epoch in range(self.init_epoch, self.epochs): print('epoch ', current_epoch) self.current_epoch = current_epoch # Forward over validation set avgloss, avgmetric = self.validation(current_epoch) self.scheduler.step(avgloss, current_epoch) # If obtained validation accuracy improvement save network in model/bestmodel.t7 if self.bestmetric < avgmetric: print('Validation metric improvement ({:.3f}) in epoch {} \n'. format(avgmetric, current_epoch)) self.bestmetric = avgmetric self.savemodel( os.path.join(self.folders['model_path'], 'bestmodel.t7')) save_ = True if self.save_rate != 0 and ( current_epoch % self.save_rate) == 0 else False # Save netowrk after self.save_rate epochs if save_: print('Saving checkpoint epoch {}\n'.format(current_epoch)) self.savemodel( os.path.join(self.folders['model_path'], 'epoch{}model.t7'.format(current_epoch))) # Forward and backward over training set self.train(current_epoch) self.valid_visualization(current_epoch, 3) # Save last model netowrk self.savemodel(os.path.join(self.folders['model_path'], 'lastmodel.t7')) ## Train function def train(self, current_epoch): ttime = time.time() data_time = AverageMeter() batch_time = AverageMeter() self.trlossavg.new_local() for key, value in self.trmetrics_avg.items(): self.trmetrics_avg[key].new_local() self.net.train() end = time.time() total_train = len(self.train_loader) for i, sample in enumerate(self.train_loader): data_time.update(time.time() - end) iteration = float(i) / total_train + current_epoch sample = self.warp_var_mod.warp_Variable(sample, self.device) images = sample['image'] outputs = self.net(images) kwarg = eval(self.losseval) loss = self.criterion(**kwarg) loss.backward() if (i + 1) % self.batch_acc == 0 or (i + 1) == total_train: self.optimizer.step() self.trlossavg.update(loss.item(), images.size(0)) for key, value in self.metrics_eval.items(): kwarg = eval(self.metrics_eval[key]) metric = self.metrics[key](**kwarg) self.trmetrics_avg[key].update(metric.item(), images.size(0)) if (i + 1) % self.batch_acc == 0 or (i + 1) == total_train: self.optimizer.zero_grad() batch_time.update(time.time() - end) end = time.time() if (i % self.print_rate) == 0: strinfo = '| Train: [{0}][{1}/{2}]\t' strinfo += 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' strinfo += 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' print(strinfo.format(current_epoch, i + 1, total_train, batch_time=batch_time, data_time=data_time), end='') for key, value in self.trmetrics_avg.items(): print('{} {:.3f} ({:.3f})\t'.format( key, value.val, value.avg), end='') print('loss {:.3f} ({:.3f})'.format(self.trlossavg.val, self.trlossavg.avg)) if self.visdom == True and (((i + 1) % self.show_rate) == 0 or ((i + 1) % total_train) == 0): info = {'loss': self.trlossavg} for key, value in self.trmetrics_avg.items(): info[key] = value for tag, value in info.items(): self.visplotter.show(tag, 'train', iteration, value.avg) self.visplotter.show(tag, 'train_mean', iteration, value.total_avg) print('|Total time: {:.3f}'.format(time.time() - ttime)) def validation(self, current_epoch): ttime = time.time() data_time = AverageMeter() batch_time = AverageMeter() self.vdlossavg.new_local() for key, value in self.vdmetrics_avg.items(): self.vdmetrics_avg[key].new_local() end = time.time() total_valid = len(self.test_loader) with torch.no_grad(): for i, sample in enumerate(self.test_loader): data_time.update(time.time() - end) iteration = float(i) / total_valid + current_epoch - 1 sample = self.warp_var_mod.warp_Variable(sample, self.device) images = sample['image'] outputs = self.net(images) kwarg = eval(self.losseval) loss = self.criterion(**kwarg) self.vdlossavg.update(loss.item(), images.size(0)) for key, value in self.metrics_eval.items(): kwarg = eval(self.metrics_eval[key]) metric = self.metrics[key](**kwarg) self.vdmetrics_avg[key].update(metric.item(), images.size(0)) batch_time.update(time.time() - end) end = time.time() if i % self.print_rate == 0: strinfo = '| Valid: [{0}][{1}/{2}]\t' strinfo += 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' strinfo += 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' print(strinfo.format( current_epoch, i + 1, total_valid, batch_time=batch_time, data_time=data_time, ), end='') for key, value in self.vdmetrics_avg.items(): print('{} {:.3f} ({:.3f})\t'.format( key, value.val, value.avg), end='') print('loss {:.3f} ({:.3f})'.format( self.vdlossavg.val, self.vdlossavg.avg)) if self.visdom == True and current_epoch != self.init_epoch and ( ((i + 1) % self.show_rate) == 0 or ((i + 1) % total_valid) == 0): info = {'loss': self.vdlossavg} for key, value in self.vdmetrics_avg.items(): info[key] = value for tag, value in info.items(): self.visplotter.show(tag, 'valid', iteration, value.avg) self.visplotter.show(tag, 'valid_mean', iteration, value.total_avg) if list(self.vdmetrics_avg.keys()): watch_metric = self.vdmetrics_avg[list( self.vdmetrics_avg.keys())[0]] else: watch_metric = self.vdlossavg print('|Total time: {:.3f}'.format(time.time() - ttime)) return self.vdlossavg.avg, watch_metric.avg def valid_visualization(self, current_epoch, index=0, save=False): with torch.no_grad(): sample = self.testdataset[index] sample['image'].unsqueeze_(0) sample = self.warp_var_mod.warp_Variable(sample, self.device) images = sample['image'] img = images[0].cpu().numpy() if self.visdom == True: self.visimshow.show('Image', img) return 1 def savemodel(self, modelpath='', killsignal=None): if modelpath == '' or killsignal is not None: print('Saving checkpoint epoch {}\n'.format(self.current_epoch)) modelpath = os.path.join( self.folders['model_path'], 'epoch{}model.t7'.format(self.current_epoch)) to_save = self.net.module if self.use_parallel else self.net state = { 'epoch': self.current_epoch, 'arch': self.arch, 'net': to_save.state_dict(), 'optimizer': self.optimizer.state_dict(), 'bestmetric': self.bestmetric } torch.save(state, modelpath) metrics_dict = { 'train_loss': self.trlossavg, 'valid_loss': self.vdlossavg } for key, value in self.trmetrics_avg.items(): metrics_dict['train_' + key] = value for key, value in self.vdmetrics_avg.items(): metrics_dict['valid_' + key] = value for tag, value in metrics_dict.items(): np.savetxt(self.folders['experiment_path'] + '/' + tag + '.txt', np.array(value.array), delimiter=',', fmt='%3.6f') if killsignal is not None: exit(-1) def loadmodel(self, modelpath): if os.path.isfile(modelpath): checkpoint = torch.load(modelpath, map_location='cpu') to_load = self.net.module if self.use_parallel else self.net to_load.load_state_dict(checkpoint['net']) self.optimizer.load_state_dict(checkpoint['optimizer']) self.current_epoch = checkpoint['epoch'] self.arch = checkpoint['arch'] self.bestmetric = checkpoint['bestmetric'] files = [ f for f in sorted(os.listdir(self.folders['experiment_path'])) if (f.find('train_') != -1 and f.find('.txt') != -1) ] for f in files: narray = np.loadtxt(os.path.join( self.folders['experiment_path'], f), delimiter=',') metric = f[6:f.find('.txt')] if metric == 'loss': self.trlossavg.load(narray, 1) if metric in self.trmetrics_avg: self.trmetrics_avg[metric].load(narray.tolist(), 1) files = [ f for f in sorted(os.listdir(self.folders['experiment_path'])) if (f.find('valid_') != -1 and f.find('.txt') != -1) ] for f in files: narray = np.loadtxt(os.path.join( self.folders['experiment_path'], f), delimiter=',') metric = f[6:f.find('.txt')] if metric == 'loss': self.vdlossavg.load(narray, 1) if metric in self.vdmetrics_avg: self.vdmetrics_avg[metric].load(narray.tolist(), 1) else: raise Exception('Model not found') def resume(self): if os.path.isdir(self.folders['model_path']): files = [ f for f in sorted(os.listdir(self.folders['model_path'])) if (f.find('epoch') != -1 and f.find('model.t7') != -1) ] if files: self.init_epoch = max( [int(f[5:f.find('model.t7')]) for f in files]) + 1 self.loadmodel( os.path.join( self.folders['model_path'], 'epoch' + str(self.init_epoch - 1) + 'model.t7')) print('Resuming on epoch' + str(self.init_epoch - 1))
class VisdomLogger(tu.AutoStateDict): """ Log metrics to Visdom. It logs scalars and scalar tensors as plots, 3D and 4D tensors as images, and strings as HTML. Args: visdom_env (str): name of the target visdom env log_every (int): batch logging freq. -1 logs on epoch ends only. prefix (str): prefix for all metrics name """ def __init__(self, visdom_env='main', log_every=10, prefix=''): super(VisdomLogger, self).__init__(except_names=['vis']) self.vis = None self.log_every = log_every self.prefix = prefix if visdom_env is not None: self.vis = Visdom(env=visdom_env) self.vis.close() def on_batch_start(self, state): iters = state['iters'] state['visdom_will_log'] = (self.log_every != -1 and iters % self.log_every == 0) @torch.no_grad() def on_batch_end(self, state): iters = state['iters'] if self.log_every != -1 and iters % self.log_every == 0: self.log(iters, state['metrics']) def on_epoch_end(self, state): self.log(state['iters'], state['metrics']) def log(self, iters, xs, store_history=[]): if self.vis is None: return for name, x in xs.items(): name = self.prefix + name if isinstance(x, (float, int)): self.vis.line(X=[iters], Y=[x], update='append', win=name, opts=dict(title=name), name=name) elif isinstance(x, str): self.vis.text(x, win=name, opts=dict(title=name)) elif isinstance(x, torch.Tensor): if x.numel() == 1: self.vis.line(X=[iters], Y=[x.item()], update='append', win=name, opts=dict(title=name), name=name) elif x.dim() == 2: self.vis.heatmap(x, win=name, opts=dict(title=name)) elif x.dim() == 3: self.vis.image(x, win=name, opts=dict(title=name, store_history=name in store_history)) elif x.dim() == 4: self.vis.images(x, win=name, opts=dict(title=name, store_history=name in store_history)) else: assert False, "incorrect tensor dim" else: assert False, "incorrect type " + x.__class__.__name__
help="num seconds between vis plotting, default just one plot") parser.add_argument("--save-loc", type=str, default='None', help="directory to save plots") return parser.parse_args() if __name__ == "__main__": args = parse_args() viz = Visdom() if not viz.check_connection(): subprocess.Popen( ["python", "-m", "visdom.server", "-p", str(args.port)]) if args.vis_interval is not None: vizes = [] while True: for v in vizes: viz.close(v) try: vizes = visdom_plot_all(viz, args.log_dir, save_loc=args.save_loc) except IOError: pass time.sleep(args.vis_interval) else: visdom_plot_all(viz, args.log_dir, save_loc=args.save_loc)
def main(args): # *** # # *** os.makedirs(args.save_folder, exist_ok=True) setproctitle.setproctitle(args.proc_name) gpus = os.getenv('CUDA_VISIBLE_DEVICES') if args.use_visdom: try: viz = Visdom(server=args.viz_ip, port=args.viz_port) assert viz.check_connection() viz.close() train_iter_plot = create_vis_plot(viz, args.proc_name + ' training') test_iter_plot = create_vis_plot(viz, args.proc_name + ' testing') except BaseException as err: raise BaseException( 'fail to connect visdom server:{}, port:{} ...'.format( args.viz_ip, args.viz_port)) args.cuda = not args.no_cuda and torch.cuda.is_available() kwargs = {'pin_memory': True} if args.cuda else {} normMean = [0.49139968, 0.48215827, 0.44653124] normStd = [0.24703233, 0.24348505, 0.26158768] normTransform = transforms.Normalize(normMean, normStd) trainTransform = transforms.Compose([ # transforms.RandomCrop(32, padding=4), transforms.RandomCrop(224, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normTransform ]) testTransform = transforms.Compose([ # transforms.RandomCrop(32,padding=4), transforms.RandomCrop(224, padding=4), transforms.ToTensor(), normTransform ]) trainLoader = DataLoader(myImageFolder(label_file=args.train_list, transform=trainTransform), batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, **kwargs) testLoader = DataLoader(myImageFolder(label_file=args.test_list, transform=testTransform), batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, **kwargs) assert not ( args.ft_net and args.resume ), 'args.resume and args.ft_net can not be used at the same time!' assert (args.ft_net or args.resume), 'args.ft_net and args.resume must have one!' start_iter = 0 if args.ft_net: net = getNetwork(args.ft_net) print('===>Initate network from pretrained model:{}'.format( args.ft_net)) if args.resume: print('===>Resuming from checkpoint:{} ..'.format(args.resume)) assert os.path.isfile( args.resume), 'Error:no checkpoint:%s found' % args.resume checkpoint = torch.load(args.resume) net = checkpoint['net'] start_iter = checkpoint['iteration'] acc = checkpoint['acc'] print('===>Resumed from checkpoint:{},start iteration:{} ..'.format( args.resume, start_iter)) print(' + Number of params: {}'.format( sum([p.data.nelement() for p in net.parameters()]))) if args.opt == 'sgd': optimizer = optim.SGD(net.fc.parameters(), lr=1e-1, momentum=0.9, weight_decay=1e-4) elif args.opt == 'adam': optimizer = optim.Adam(net.fc.parameters(), weight_decay=1e-4) elif args.opt == 'rmsprop': optimizer = optim.RMSprop(net.fc.parameters(), weight_decay=1e-4) if args.cuda: net = net.cuda() if len(gpus) > 1: net = torch.nn.DataParallel(net) cudnn.benchmark = True train_batch_iterator = iter(trainLoader) for iteration in range(start_iter, args.max_iter): adjust_opt(args.opt, optimizer, iteration) train_batch = next(train_batch_iterator) train(args, iteration, net, train_batch, optimizer, viz, train_iter_plot) if iteration % args.test_interval == 0: test_acc = test(args, iteration, net, testLoader, viz, test_iter_plot) if iteration % args.snapshot == 0: state = { 'net': net.module if len(gpus) > 1 else net, 'iteration': iteration, 'acc': test_acc } snapshot = ospj(args.save_folder, args.proc_name + '_{}.pth'.format(iteration)) torch.save(state, snapshot) print('Saving state, iter: {}, to {}'.format(iteration, snapshot))
class Experiment(object): """ Experiment class """ def __init__(self, name, hparams, desc=None): """ Args: name (string): the name of the experiment hparams (object): the hypermarameters used for this experiment """ self.name = name self.desc = desc self.hparams = hparams self.metrics = defaultdict(Metric) self.timestamp_start = datetime.now() self.timestamp_update = datetime.now() self.last_update = time.time() self.viz = Visdom() self.viz.close() self.vis_params() if desc is not None: self.vis_desc() def update_plots(self): for exp_name, metric in self.metrics.items(): metric.update_plot() # self.save_experiment() def vis_params(self): lines = [] for param, value in self.hparams.items(): lines.append("{}: {}".format(param, value)) self.viz.text("<pre>{}</pre>".format("\n".join(lines)), opts=dict( width=max([len(x) for x in lines]) * 10, height=len(lines) * 20, )) def vis_desc(self): self.viz.text("<pre>{}</pre>".format(self.desc), opts=dict( width=max([len(x) for x in self.desc.split("\n")]) * 8.5, height=len(self.desc.split("\n")) * 20, )) def add_metric(self, metric): """ Add a metric to the experiment Args: metric (Metric): a metric object Returns: """ metric.vic_context = self.viz self.metrics[metric.name] = metric def get_score(self, metric, tag): return self.metrics[metric]._values[tag][-1] def save_experiment(self): """ Implement a saving mechanism (in text, csv or a database) Returns: """ self.timestamp_update = datetime.now() self.db.update({ 'name': self.name, 'desc': self.desc, 'hparams': self.hparams, 'metrics': self.metrics, 'timestamp_start': self.timestamp_start, 'timestamp_update': self.timestamp_update, 'last_update': self.last_update }, doc_ids=[self.db_record]) def visualize_experiment(self): raise NotImplementedError