def next(self): if self._index >= len(self._batches): raise StopIteration s = slice(*(self._batches[self._index])) Xbuf = self._X[s,:] if Xbuf.dtype == 'uint8': if self._Xbuf == None: self._Xbuf = bm.empty((self._batchsize,self._X.shape[1])) self._Xbuf[:s.stop-s.start,:] = Xbuf # copy Xbuf = self._Xbuf[:s.stop-s.start,:] # point to copy bm.imul(Xbuf,1./255) batch = DataFold(Xbuf,self._Y[s,:],self._S[s,:] if self._S != None else None) self._index += 1 return batch
def next(self): if self._index >= len(self._batches): raise StopIteration s = slice(*(self._batches[self._index])) Xbuf = self._X[s, :] if Xbuf.dtype == 'uint8': if self._Xbuf == None: self._Xbuf = bm.empty((self._batchsize, self._X.shape[1])) self._Xbuf[:s.stop - s.start, :] = Xbuf # copy Xbuf = self._Xbuf[:s.stop - s.start, :] # point to copy bm.imul(Xbuf, 1. / 255) batch = DataFold(Xbuf, self._Y[s, :], self._S[s, :] if self._S != None else None) self._index += 1 return batch
def _collect_stats_on_batch(self,batch): ''' Give a particular fold (training/testing) this evaluates the model using the current fold, and collects statistics about how the model is performing. ''' X,Y = batch model = self.trainer().model H = model.eval(batch,want_hidden=True) stats = {} stats["X"] = bm.as_numpy(X).copy() stats["Y"] = bm.as_numpy(Y).copy() if not X is Y else stats["X"] stats["S"] = bm.as_numpy(batch.S).copy() if batch.S != None else zeros((0,1),dtype='float32') stats["H"] = [bm.as_numpy(Hi).copy() for Hi in H] # make a copy of hidden activations stats["loss"] = model.loss(H[-1],Y) # scalar loss value stats["regularizer"] = model.regularizer(H) # scalar hidden unit regularization penalty stats["penalty"] = model.penalty() # scalar weight penalty if self.trainer().task() == "classification": stats["error rate"] = 100*count_nonzero(array(argmax(H[-1],axis=1)) != argmax(Y,axis=1)) / float(batch.size) return stats
def __init__(self,X,Y,Xshape=None,Yshape=None,Xrange=None,Yrange=None,shuffle=True,max_batchsize=1): if shuffle: perm = random.permutation(X.shape[0]) X[:] = X[perm] Y[:] = Y[perm] self._X = bm.asarray(X) self._Y = bm.asarray(Y) self._size = X.shape[0] self.max_batchsize = max_batchsize self.Xshape = Xshape or (1,X.shape[1]) self.Yshape = Yshape or (1,Y.shape[1]) self.Xdim = X.shape[1] self.Ydim = Y.shape[1] self.Xrange = Xrange or (X.min(axis=0),X.max(axis=0)) self.Yrange = Yrange or (Y.min(axis=0),Y.max(axis=0)) rs = self._rowslice(0,self._size) self.train = DataFold(X[rs,:],Y[rs,:]) self.valid = DataFold(X[0:0,:],Y[0:0,:]) self.test = DataFold(X[0:0,:],Y[0:0,:])
def log(self, event, stats): if self._direction == "input": # Filters going into first layer of hidden units W = self._model.weights[0].W W = bm.as_numpy(W).copy() W = W.reshape(self._featshape + tuple([-1])) else: # Templates going out of final layer of hidden units W = self._model.weights[-1].W W = bm.as_numpy(W).copy().transpose() W = W.reshape(self._featshape + tuple([-1])) self._feat = W # col[i] contains weights entering unit i in first hidden layer self._featrange = (min(W.ravel()), max(W.ravel())) self._dirty = True if event == "epoch" and self._sorted and (stats["epoch"] < 5): # Sort by decreasing L2 norm ranks = [-sum(self._feat[:, :, j].ravel() ** 2) for j in range(self._feat.shape[2])] self._ordering = argsort(ranks) if self._ordering != None: self._feat = self._feat[:, :, self._ordering]
def __init__(self, X, Y, Xshape=None, Yshape=None, Xrange=None, Yrange=None, shuffle=True, max_batchsize=1): if isinstance(X, dict): Xtest = X['test'] Ytest = Y['test'] X = X['train'] Y = Y['train'] else: Xtest = X[0, :] # empty Ytest = Y[0, :] # empty if shuffle: perm = random.permutation(X.shape[0]) X = take(X, perm, axis=0) if not (X is Y): Y = take(Y, perm, axis=0) t0 = now() self._X = bm.asarray(X) self._Y = bm.asarray(Y) if not (X is Y) else self._X self._Xtest = bm.asarray(Xtest) self._Ytest = bm.asarray(Ytest) if not (X is Y) else self._Xtest if bm.backend_name == "gnumpy": print "Host->Device transfer of dataset took %.3fs" % (now() - t0) self._size = X.shape[0] self._Xrescale = (1., 0.) #scale,bias self.max_batchsize = max_batchsize self.Xshape = Xshape or (1, X.shape[1]) self.Yshape = Yshape or (1, Y.shape[1]) self.Xdim = X.shape[1] self.Ydim = Y.shape[1] self.Xrange = Xrange or (X.min(axis=0), X.max(axis=0)) self.Yrange = Yrange or (Y.min(axis=0), Y.max(axis=0)) if not isscalar(self.Xrange[0]): self.Xrange = (bm.asarray(self.Xrange[0]).reshape( (1, -1)), bm.asarray(self.Xrange[1]).reshape((1, -1))) if not isscalar(self.Yrange[0]): self.Yrange = (bm.asarray(self.Yrange[0]).reshape( (1, -1)), bm.asarray(self.Yrange[1]).reshape((1, -1))) rs = self._rowslice(0, self._size) self.train = DataFold(self._X[rs, :], self._Y[rs, :]) self.valid = DataFold(self._X[0:0, :], self._Y[0:0, :]) self.test = DataFold(self._Xtest, self._Ytest)
def log(self,event,stats): if self._direction == 'input': # Filters going into first layer of hidden units W,b = self._model.weights[0] W = bm.as_numpy(W).copy().transpose() #W += bm.as_numpy(b).transpose() W = W.reshape(tuple([-1]) + self._featshape) else: # Templates going out of final layer of hidden units W = self._model.weights[-1].W W = bm.as_numpy(W).copy() W = W.reshape(tuple([-1]) + self._featshape) self._feat = W # col[i] contains weights entering unit i in first hidden layer self._featrange = (W.ravel().min(),W.ravel().max()) self._dirty = True if event == 'epoch': self._update_count += 1 if self._sorted and self._update_count <= 25: # Sort by decreasing variance ranks = [-var(self._feat[j,:,:].ravel()) for j in range(self._feat.shape[0])] self._ordering = argsort(ranks) if self._ordering != None: self._feat = self._feat[self._ordering,:,:]
def _collect_stats_on_batch(self, batch): """ Give a particular fold (training/testing) this evaluates the model using the current fold, and collects statistics about how the model is performing. """ X, Y = batch model = self.trainer().model H = model.eval(X, want_hidden=True) stats = {} stats["H"] = [bm.as_numpy(Hi).copy() for Hi in H] # make a copy of hidden activations stats["loss"] = model.loss(H[-1], Y) # scalar loss value stats["regularizer"] = model.regularizer(H) # scalar hidden unit regularization penalty stats["penalty"] = model.penalty() # scalar weight penalty if self.trainer().task() == "classification": stats["error rate"] = ( 100 * count_nonzero(array(argmax(H[-1], axis=1)) != argmax(Y, axis=1)) / float(batch.size) ) return stats
def __init__(self,X,Y,Xshape=None,Yshape=None,Xrange=None,Yrange=None,shuffle=True,max_batchsize=1): if isinstance(X,dict): Xtest = X['test'] Ytest = Y['test'] X = X['train'] Y = Y['train'] else: Xtest = X[0,:] # empty Ytest = Y[0,:] # empty if shuffle: perm = random.permutation(X.shape[0]) X = take(X,perm,axis=0) if not (X is Y): Y = take(Y,perm,axis=0) t0 = now() self._X = bm.asarray(X) self._Y = bm.asarray(Y) if not (X is Y) else self._X self._Xtest = bm.asarray(Xtest) self._Ytest = bm.asarray(Ytest) if not (X is Y) else self._Xtest if bm.backend_name == "gnumpy": print "Host->Device transfer of dataset took %.3fs" % (now()-t0) self._size = X.shape[0] self._Xrescale = (1.,0.) #scale,bias self.max_batchsize = max_batchsize self.Xshape = Xshape or (1,X.shape[1]) self.Yshape = Yshape or (1,Y.shape[1]) self.Xdim = X.shape[1] self.Ydim = Y.shape[1] self.Xrange = Xrange or (X.min(axis=0),X.max(axis=0)) self.Yrange = Yrange or (Y.min(axis=0),Y.max(axis=0)) if not isscalar(self.Xrange[0]): self.Xrange = (bm.asarray(self.Xrange[0]).reshape((1,-1)),bm.asarray(self.Xrange[1]).reshape((1,-1))) if not isscalar(self.Yrange[0]): self.Yrange = (bm.asarray(self.Yrange[0]).reshape((1,-1)),bm.asarray(self.Yrange[1]).reshape((1,-1))) rs = self._rowslice(0,self._size) self.train = DataFold(self._X[rs,:],self._Y[rs,:]) self.valid = DataFold(self._X[0:0,:],self._Y[0:0,:]) self.test = DataFold(self._Xtest,self._Ytest)
def __init__(self, master, size, dpi, data): Figure.__init__( self, figsize=(size[0] / dpi, size[1] / dpi), dpi=dpi, facecolor="w", edgecolor="b", frameon=True, linewidth=0, ) FigureCanvas(self, master=master) self.master = master self._dirty = True self._fold = "test" if data["test"].size > 0 else "train" self._indices = rnd.sample(arange(data[self._fold].size), minimum(data[self._fold].size, 50)) # 256)) self._targets = ( bm.as_numpy(data[self._fold].Y[self._indices, :]) .transpose() .reshape(data.Yshape + tuple([len(self._indices)])) ) self._outputs = None self._outshape = data.Yshape self._outrange = data.Yrange self.add_subplot(111, axisbg="w")
def rescale(self, Xrange, Yrange): ''' Rescales the entire dataset so that all inputs X lie within (Xrange[0],Xrange[1]) and all targets Y lie within (Yrange[0],Yrange[1]). The same scaling factor is applied to all folds. ''' if Xrange != self.Xrange and self._X.dtype != 'uint8': Xscale = self.Xrange[1] - self.Xrange[0] if isscalar(Xscale): Xscale = (Xrange[1] - Xrange[0]) / maximum(1e-5, Xscale) else: bm.maximum(Xscale, 1e-5, out=Xscale) bm.reciprocal(Xscale, out=Xscale) bm.multiply(Xscale, Xrange[1] - Xrange[0], out=Xscale) bm.isub(self._X, self.Xrange[0]) bm.imul(self._X, Xscale) bm.iadd(self._X, Xrange[0]) if Yrange != self.Yrange and not (self._X is self._Y): Yscale = self.Yrange[1] - self.Yrange[0] if isscalar(Yscale): Yscale = (Yrange[1] - Yrange[0]) / maximum(1e-5, Yscale) else: bm.maximum(Yscale, 1e-5, out=Yscale) bm.reciprocal(Yscale, out=Yscale) bm.multiply(Yscale, Yrange[1] - Yrange[0], out=Yscale) bm.isub(self._Y, self.Yrange[0]) bm.imul(self._Y, Yscale) bm.iadd(self._Y, Yrange[0]) self.Xrange = Xrange self.Yrange = Yrange
def __init__(self,trainer,window_size): Tk.Frame.__init__(self) self._unique_id = 0 # Set up a 2x2 grid, where each cell will have its own kind of figure self.master.rowconfigure(0,weight=1) self.master.rowconfigure(1,weight=1) self.master.rowconfigure(2,weight=1) self.master.rowconfigure(3,weight=1) self.master.rowconfigure(4,weight=1) self.master.columnconfigure(0,weight=1) self.master.columnconfigure(1,weight=1) self.master.columnconfigure(2,weight=1) dpi = 80.0 self.plots = {} if window_size == "compact": col0_wd = 300 col1_wd = 300 row0_ht = 200 row1_ht = 100 else: col0_wd = 900 col1_wd = 770 row0_ht = 345 row1_ht = 470 # Add error plot in top-left cell self.plots["errors"] = TrainingReportErrorPlot(self.master,(col0_wd,row0_ht),dpi,trainer.task()) self.plots["errors"].canvas.get_tk_widget().grid(row=0,column=0,sticky=Tk.N+Tk.S+Tk.E+Tk.W) has_input_feat = trainer.data.Xshape[0] > 1 and trainer.data.Xshape[1] has_output_feat = trainer.data.Yshape[0] > 1 and trainer.data.Yshape[1] # Input feature grid in top-right cell if has_input_feat: self.plots["feat_in"] = TrainingReportFeatureGrid(self.master,(col1_wd,row0_ht),dpi,trainer.model,trainer.data.Xshape,"input") self.plots["feat_in"].canvas.get_tk_widget().grid(row=0,column=1,columnspan=(1 if has_output_feat else 2),sticky=Tk.N+Tk.S+Tk.E+Tk.W) # Output feature grid in top-right-right cell if trainer.data.Yshape[0] > 1 and trainer.data.Yshape[1]: self.plots["feat_out"] = TrainingReportFeatureGrid(self.master,(col1_wd,row0_ht),dpi,trainer.model,trainer.data.Yshape,"output") self.plots["feat_out"].canvas.get_tk_widget().grid(row=0,column=(2 if has_input_feat else 1),columnspan=(1 if has_input_feat else 2),sticky=Tk.N+Tk.S+Tk.E+Tk.W) # *Weight* statistics in bottom-left cell weights_ref = weakref.ref(trainer.model.weights) get_weightmats = lambda event,stats: [bm.as_numpy(abs(layer.W)) for layer in weights_ref()] #weight_percentiles = list(100*(1-linspace(0.1,.9,10)**1.5)) weight_percentiles = list(100*(1-linspace(0.05,.95,20))) self.plots["wstats"] = TrainingReportPercentiles(self.master,(col0_wd,row1_ht),dpi,get_weightmats,weight_percentiles,True,title="W") self.plots["wstats"].canvas.get_tk_widget().grid(row=1,column=0,sticky=Tk.N+Tk.S+Tk.E+Tk.W) # *Hidden activity* statistics in bottom-right cell get_hidden = lambda event,stats: stats["train"]["H"] #hidden_percentiles = list(100*(1-linspace(0.1,.9,10)**1.5)) hidden_percentiles = list(100*(1-linspace(0.05,.95,20))) ranges = [layer.f.actual_range() for layer in trainer.model._cfg[1:]] self.plots["hstats"] = TrainingReportPercentiles(self.master,(col0_wd,row1_ht),dpi,get_hidden,hidden_percentiles,True,ranges=ranges,title="H") # For problems with 2D output, draw the target and the reconstruction side by side if trainer.data.Yshape[0] > 1 and trainer.data.Yshape[1]: if trainer.data["test"].size > 0: self.plots["recons_tr"] = TrainingReportReconstructGrid(self.master,(col1_wd,row1_ht),dpi,trainer.data,"train") self.plots["recons_tr"].canvas.get_tk_widget().grid(row=1,column=1,rowspan=3,sticky=Tk.N+Tk.S+Tk.E+Tk.W) self.plots["recons_te"] = TrainingReportReconstructGrid(self.master,(col1_wd,row1_ht),dpi,trainer.data,"test") self.plots["recons_te"].canvas.get_tk_widget().grid(row=1,column=2,rowspan=3,sticky=Tk.N+Tk.S+Tk.E+Tk.W) else: self.plots["recons_tr"] = TrainingReportReconstructGrid(self.master,(col1_wd,row1_ht),dpi,trainer.data,"train") self.plots["recons_tr"].canvas.get_tk_widget().grid(row=1,column=1,columnspan=2,rowspan=3,sticky=Tk.N+Tk.S+Tk.E+Tk.W) if self.plots.has_key("hstats"): self.plots["hstats"].canvas.get_tk_widget().grid(row=2,column=0,sticky=Tk.N+Tk.S+Tk.E+Tk.W) else: if self.plots.has_key("hstats"): self.plots["hstats"].canvas.get_tk_widget().grid(row=1,column=1,sticky=Tk.N+Tk.S+Tk.E+Tk.W) ''' num_activityplots = min(3,trainer.model.numlayers()-1) for k in range(num_activityplots): name = 'activity%i' % k self.plots[name] = TrainingReportActivityPlot(self.master,(col0_wd/dpi,100/dpi),dpi,k,xaxis=(k==num_activityplots-1)) if self.plots.has_key("recons_tr"): self.plots[name].canvas.get_tk_widget().grid(row=1+k,column=0,columnspan=1,sticky=Tk.N+Tk.S+Tk.W+Tk.E) else: self.plots[name].canvas.get_tk_widget().grid(row=1+k,column=0,columnspan=3,sticky=Tk.N+Tk.S+Tk.W+Tk.E) ''' #self.master.geometry('2000x900+%d+%d' % (0,100)) self.master.geometry('600x300+%d+%d' % (0,100)) self.master.title("Training Report") self.update() self._redraw_interval = 500
def __init__(self, trainer): Tk.Frame.__init__(self) self._unique_id = 0 # Set up a 2x2 grid, where each cell will have its own kind of figure self.master.rowconfigure(0, weight=1) self.master.rowconfigure(1, weight=1) self.master.rowconfigure(2, weight=1) self.master.columnconfigure(0, weight=1) self.master.columnconfigure(1, weight=1) self.master.columnconfigure(2, weight=1) dpi = 80.0 self.plots = {} # col0_wd = 320 # col1_wd = 870 # row0_ht = 400 # row1_ht = 220 col0_wd = 300 col1_wd = 300 row0_ht = 200 row1_ht = 100 # Add error plot in top-left cell self.plots["errors"] = TrainingReportErrorPlot(self.master, (col0_wd, row0_ht), dpi, trainer.task()) self.plots["errors"].canvas.get_tk_widget().grid(row=0, column=0, sticky=Tk.N + Tk.S + Tk.E + Tk.W) # Input feature grid in top-right cell if trainer.data.Xshape[0] > 1 and trainer.data.Xshape[1]: self.plots["feat_in"] = TrainingReportFeatureGrid( self.master, (col1_wd, row0_ht), dpi, trainer.model, trainer.data.Xshape, "input" ) self.plots["feat_in"].canvas.get_tk_widget().grid(row=0, column=1, sticky=Tk.N + Tk.S + Tk.E + Tk.W) # Output feature grid in top-right-right cell if trainer.data.Yshape[0] > 1 and trainer.data.Yshape[1]: self.plots["feat_out"] = TrainingReportFeatureGrid( self.master, (col1_wd, row0_ht), dpi, trainer.model, trainer.data.Yshape, "output" ) self.plots["feat_out"].canvas.get_tk_widget().grid( row=0, column=(2 if self.plots.has_key("feat_in") else 1), sticky=Tk.N + Tk.S + Tk.E + Tk.W ) # *Weight* statistics in bottom-left cell weights_ref = weakref.ref(trainer.model.weights) get_weightmats = lambda event, stats: [bm.as_numpy(abs(layer.W)) for layer in weights_ref()] weight_percentiles = list(100 * (1 - linspace(0.1, 0.9, 10) ** 1.5)) self.plots["wstats"] = TrainingReportPercentiles( self.master, (col0_wd, row1_ht), dpi, get_weightmats, weight_percentiles, True, title="W" ) self.plots["wstats"].canvas.get_tk_widget().grid(row=1, column=0, sticky=Tk.N + Tk.S + Tk.E + Tk.W) # *Hidden activity* statistics in bottom-right cell get_hidden = lambda event, stats: stats["train"]["H"] hidden_percentiles = list(100 * (1 - linspace(0.1, 0.9, 10) ** 1.5)) ranges = [layer.f.actual_range() for layer in trainer.model._cfg[1:]] self.plots["hstats"] = TrainingReportPercentiles( self.master, (col0_wd, row1_ht), dpi, get_hidden, hidden_percentiles, False, ranges=ranges, title="H" ) self.plots["hstats"].canvas.get_tk_widget().grid(row=1, column=1, sticky=Tk.N + Tk.S + Tk.E + Tk.W) # For problems with 2D output, draw the target and the reconstruction side by side if trainer.data.Yshape[0] > 1 and trainer.data.Yshape[1]: self.plots["recons"] = TrainingReportReconstructGrid(self.master, (col1_wd, row1_ht), dpi, trainer.data) self.plots["recons"].canvas.get_tk_widget().grid( row=1, column=1, rowspan=2, sticky=Tk.N + Tk.S + Tk.E + Tk.W ) self.master.geometry("+%d+%d" % (0, 180)) self.master.title("Training Report") self.update() self._redraw_interval = 500
def rescale(self,Xrange,Yrange): ''' Rescales the entire dataset so that all inputs X lie within (Xrange[0],Xrange[1]) and all targets Y lie within (Yrange[0],Yrange[1]). The same scaling factor is applied to all folds. ''' if Xrange != self.Xrange and self._X.dtype != 'uint8': Xscale = self.Xrange[1]-self.Xrange[0] if isscalar(Xscale): Xscale = (Xrange[1]-Xrange[0]) / maximum(1e-5,Xscale) else: bm.maximum(Xscale,1e-5,out=Xscale) bm.reciprocal(Xscale,out=Xscale) bm.multiply(Xscale,Xrange[1]-Xrange[0],out=Xscale) bm.isub(self._X,self.Xrange[0]) bm.imul(self._X,Xscale) bm.iadd(self._X,Xrange[0]) if Yrange != self.Yrange and not (self._X is self._Y): Yscale = self.Yrange[1]-self.Yrange[0] if isscalar(Yscale): Yscale = (Yrange[1]-Yrange[0]) / maximum(1e-5,Yscale) else: bm.maximum(Yscale,1e-5,out=Yscale) bm.reciprocal(Yscale,out=Yscale) bm.multiply(Yscale,Yrange[1]-Yrange[0],out=Yscale) bm.isub(self._Y,self.Yrange[0]) bm.imul(self._Y,Yscale) bm.iadd(self._Y,Yrange[0]) self.Xrange = Xrange self.Yrange = Yrange