def fit_generator(self, x, y, generator, loss_func='categorical_crossentropy', optimizer=SGD(lr=0.01, momentum=0.9), clip=None, callbacks=[], transformer=None, verbose=1): # Train for batch_x, batch_y in generator.generate(x, y): batch_x = to_list(batch_x) batch_y = to_list(batch_y) t1 = time.time() # Compile optimization function if not self._f_optimize_: # Train memory usage batch_size = len(batch_x[0]) print "Training", self._show_memory_usage(self._effective_layers_, batch_size) timer = Timer() target_dim_list = self._get_target_dim_list( batch_x, batch_y, transformer) self._f_optimize_ = self.get_optimization_func( target_dim_list, loss_func, optimizer, clip) timer.show("Compiling f_optimize time:") # Compile for callback timer = Timer() if callbacks is not None: callbacks = to_list(callbacks) for callback in callbacks: callback.compile(self) timer.show("Compiling callbacks time:") if transformer: (batch_x, batch_y) = transformer.transform(batch_x, batch_y) batch_x = format_data_list(batch_x) batch_y = format_data_list(batch_y) # Callback for callback in callbacks: if (self.iter_ % callback.call_freq_ == 0): print callback.call() in_list = batch_x + batch_y + [1.] # training phase loss = self._f_optimize_(*in_list)[0] self._iter_ += 1 t2 = time.time() self._tr_time_ += (t2 - t1) sys.stdout.write( "iteration: %d loss: %f time per batch: %.2f \r" % (self._iter_, loss, t2 - t1)) sys.stdout.flush()
def train_on_batch(self, batch_x, batch_y, loss_func='categorical_crossentropy', optimizer=SGD(lr=0.01, momentum=0.9), clip=None, callbacks=[], transformer=None, recompile=False): """Train model on single batch data. Args: batch_x: ndarray | list of ndarray. batch_y: ndarray | list of ndarray. loss_func: str | function. optimizer: optimization object. clip: real value. callbacks: list of Callback object. transformer: Transformation object. recompile: bool. Recompile the optimize function or not. Returns: None. (All trained models, results should be saved using callbacks.) """ batch_x = to_list(batch_x) batch_y = to_list(batch_y) # Format data batch_x = format_data_list(batch_x) batch_y = format_data_list(batch_y) # Compile optimization function if (not self._f_optimize_) or (recompile): timer = Timer() target_dim_list = self._get_target_dim_list(batch_x, batch_y, transformer) self._f_optimize_ = self.get_optimization_func(target_dim_list, loss_func, optimizer, clip) timer.show("Compiling f_optimize time:") batch_size = len(batch_x[0]) self._show_memory_usage(self._effective_layers_, batch_size) # Compile for callback timer = Timer() if callbacks is not None: callbacks = to_list(callbacks) for callback in callbacks: callback.compile(self) timer.show("Compiling callbacks time:") for callback in callbacks: if (self.iter_ % callback.call_freq_ == 0): print self.iter_, 'th iteration:' callback.call() # Train t1 = time.time() if transformer: (batch_x, batch_y) = transformer.transform(batch_x, batch_y) batch_x = format_data_list(batch_x) batch_y = format_data_list(batch_y) in_list = batch_x + batch_y + [1.] # training phase loss = self._f_optimize_(*in_list)[0] self._iter_ += 1 t2 = time.time() self._tr_time_ += (t2 - t1) return loss
def _evaluate(self, x, y, eval_type): # get metric losses node loss_nodes = [] for metric in self._metrics_: # if use default objective if type(metric) is str: assert len(self._md_.out_nodes_)==len(self._md_.gt_nodes_), "If you are using default objectives, " \ + "out_node of out_layers must match ground truth!" loss_node = sum([obj.get(metric)(pred_node, gt_node) for pred_node, gt_node in zip(self._md_.out_nodes_, self._md_.gt_nodes_)]) # if user define their objective function elif isfunction(metric): loss_node = metric(self._md_) else: loss_node = metric loss_nodes.append(loss_node) # compile evaluation function if not hasattr(self, '_f_evaluate'): print 'compiling evaluation function ..' inputs = self._md_.in_nodes_ + self._md_.gt_nodes_ + [self._md_.tr_phase_node_] self._f_evaluate = K.function_no_given(inputs, loss_nodes) print 'compile finished. ' # calculate metric values t1 = time.time() if self._generator_: generator = self._generator_ else: generator = self._DefaultGenerator(self._batch_size_) n_all = 0. cnt= 0. metric_vals = np.zeros(len(self._metrics_)) batch_num = sum(1 for it in generator.generate(x, y)) for batch_x, batch_y in generator.generate(x, y): batch_x = to_list(batch_x) batch_y = to_list(batch_y) curr_batch_size = batch_x[0].shape[0] n_all += curr_batch_size cnt += 1. if self._transformer_: (batch_x, batch_y) = self._transformer_.transform(batch_x, batch_y) batch_x = format_data_list(batch_x) batch_y = format_data_list(batch_y) in_list = batch_x + batch_y + [0.] batch_metric_vals = np.array(self._f_evaluate(*in_list)) metric_vals += batch_metric_vals * curr_batch_size if self._verbose_==1: self._print_progress(batch_num, cnt) metric_vals /= n_all # timer t2 = time.time() # print results self._print_time_results(eval_type, self._metrics_, metric_vals, t2-t1)
def generate(self, x, y): x = to_list(x) y = to_list(y) N = len(x[0]) batch_num = int(np.ceil(float(N) / self._batch_size_)) # evaluate for each batch for i1 in xrange(batch_num): curr_batch_size = min((i1+1)*self._batch_size_, N) - i1*self._batch_size_ batch_x = [e[i1*self._batch_size_ : min((i1+1)*self._batch_size_, N)] for e in x] batch_y = [e[i1*self._batch_size_ : min((i1+1)*self._batch_size_, N)] for e in y] yield batch_x, batch_y
def __init__(self, in_layers, out_layers, any_layers=[]): super(Model, self).__init__(in_layers) # out layers out_layers = to_list(out_layers) self._out_layers_ = out_layers self._out_nodes_ = [layer.output_ for layer in self._out_layers_] # inter layers any_layers = to_list(any_layers) self._any_layers_ = any_layers self._any_nodes_ = [layer.output_ for layer in self._any_layers_]
def fit_generator(self, x, y, generator, loss_func='categorical_crossentropy', optimizer=SGD(lr=0.01, momentum=0.9), clip=None, callbacks=[], transformer=None, verbose=1): # Train for batch_x, batch_y in generator.generate(x, y): batch_x = to_list(batch_x) batch_y = to_list(batch_y) t1 = time.time() # Compile optimization function if not self._f_optimize_: # Train memory usage batch_size = len(batch_x[0]) print "Training", self._show_memory_usage(self._effective_layers_, batch_size) timer = Timer() target_dim_list = self._get_target_dim_list(batch_x, batch_y, transformer) self._f_optimize_ = self.get_optimization_func(target_dim_list, loss_func, optimizer, clip) timer.show("Compiling f_optimize time:") # Compile for callback timer = Timer() if callbacks is not None: callbacks = to_list(callbacks) for callback in callbacks: callback.compile(self) timer.show("Compiling callbacks time:") if transformer: (batch_x, batch_y) = transformer.transform(batch_x, batch_y) batch_x = format_data_list(batch_x) batch_y = format_data_list(batch_y) # Callback for callback in callbacks: if (self.iter_ % callback.call_freq_ == 0): print callback.call() in_list = batch_x + batch_y + [1.] # training phase loss = self._f_optimize_(*in_list)[0] self._iter_ += 1 t2 = time.time() self._tr_time_ += (t2 - t1) sys.stdout.write("iteration: %d loss: %f time per batch: %.2f \r" % (self._iter_, loss, t2-t1)) sys.stdout.flush()
def get_observe_forward_func(self, observe_nodes): observe_nodes = to_list(observe_nodes) inputs = self.in_nodes_ + [self.tr_phase_node_] timer = Timer() f_observe_forward = K.function_no_given(inputs, observe_nodes) timer.show("Compiling f_observe_forward time:") return f_observe_forward
def __init__(self, in_layers): # reset global id reset_id_to_zero() # inlayers in_layers = to_list(in_layers) self._in_layers_ = in_layers self._in_nodes_ = [layer.output_ for layer in self._in_layers_] # find all nodes using BFT self._id_list_, self._layer_list_ = BFT(self._in_layers_) # get params by traveling all layers self._params_ = [] for layer in self._layer_list_: self._params_ += layer.params_ # get inner updates by traveling all layers self._inner_updates_ = [] for layer in self._layer_list_: if hasattr(layer, 'inner_updates_'): self._inner_updates_ += layer.inner_updates_ # sum regs by traveling all layers self._reg_value_ = 0. for layer in self._layer_list_: self._reg_value_ += layer.reg_value_ # tr_phase_node self._tr_phase_node_ = K.common_tr_phase_node # time self._tr_time_ = 0. self._epoch_ = 0
def __init__(self, tr_x=None, tr_y=None, va_x=None, va_y=None, te_x=None, te_y=None, batch_size=100, call_freq=3, metrics=['categorical_error'], generator=None, transformer=None, dump_path=None, type='epoch', verbose=1, is_logging=False): # init values super(Validation, self).__init__(call_freq, type) self._batch_size_ = batch_size self._metrics_ = to_list(metrics) self._generator_ = generator self._transformer_ = transformer self._dump_path_ = dump_path self._type_ = type self._verbose_ = verbose self._r_ = self._init_result() self._is_logging_ = is_logging # judge if train or valid or test data exists self._is_tr_ = self._is_data(tr_x, tr_y) self._is_va_ = self._is_data(va_x, va_y) self._is_te_ = self._is_data(te_x, te_y) self._tr_x_ = tr_x self._tr_y_ = tr_y self._va_x_ = va_x self._va_y_ = va_y self._te_x_ = te_x self._te_y_ = te_y
def __init__(self, tr_x=None, tr_y=None, va_x=None, va_y=None, te_x=None, te_y=None, batch_size=100, call_freq=3, metrics=['categorical_error'], dump_path='validation.p', verbose=1): # init values self._batch_size_ = batch_size self._call_freq_ = call_freq self._metrics_ = to_list(metrics) self._dump_path_ = dump_path self._verbose_ = verbose self._r_ = self._init_result() # judge if train or valid or test data exists self._is_tr_ = self._is_data(tr_x, tr_y) self._is_va_ = self._is_data(va_x, va_y) self._is_te_ = self._is_data(te_x, te_y) # format data, to list if self._is_tr_: self._tr_x_ = [K.format_data(e) for e in to_list(tr_x)] self._tr_y_ = [K.format_data(e) for e in to_list(tr_y)] if self._is_va_: self._va_x_ = [K.format_data(e) for e in to_list(va_x)] self._va_y_ = [K.format_data(e) for e in to_list(va_y)] if self._is_te_: self._te_x_ = [K.format_data(e) for e in to_list(te_x)] self._te_y_ = [K.format_data(e) for e in to_list(te_y)]
def get_observe_backward_func(self, observe_nodes): if self.gt_nodes_ is None: raise Exception("You must call set_gt_nodes method before call observe_backward method!") observe_nodes = to_list(observe_nodes) inputs = self.in_nodes_ + self.gt_nodes_ + [self.tr_phase_node_] timer = Timer() f_observe_backward = K.function_no_given(inputs, observe_nodes) timer.show("Compiling f_observe_backward time:") return f_observe_backward
def __init__(self, in_layers=[], out_layers=[], any_layers=[]): self._in_layers_ = to_list(in_layers) self._out_layers_ = to_list(out_layers) self._any_layers_ = to_list(any_layers) self._gt_nodes_ = None self._f_predict = None self._epoch_ = 0 self._iter_= 0 self._tr_time_ = 0. self._tr_phase_node_ = K.common_tr_phase_node self._effective_layers_ = self.get_effective_layers(self._in_layers_, self._out_layers_) self._check_duplicate_name(self._effective_layers_) self._trainable_table_ = self._init_trainable_table(self._effective_layers_) self._f_optimize_ = None
def train_on_batch_with_func(self, func, batch_x, batch_y): """Train model on batch data. Args: func: function. batch_x: ndarray | list of ndarray. batch_y: ndarray | list of ndarray. """ batch_x = to_list(batch_x) batch_y = to_list(batch_y) # Format data batch_x = format_data_list(batch_x) batch_y = format_data_list(batch_y) in_list = batch_x + batch_y + [1.] # training phase loss = func(*in_list)[0] self._set_iter(self.iter_ + 1) return loss
def generate(self, x, y): x = to_list(x) y = to_list(y) N = len(x[0]) batch_num = int(np.ceil(float(N) / self._batch_size_)) # evaluate for each batch for i1 in xrange(batch_num): curr_batch_size = min( (i1 + 1) * self._batch_size_, N) - i1 * self._batch_size_ batch_x = [ e[i1 * self._batch_size_:min((i1 + 1) * self._batch_size_, N)] for e in x ] batch_y = [ e[i1 * self._batch_size_:min((i1 + 1) * self._batch_size_, N)] for e in y ] yield batch_x, batch_y
def set_layers_trainability(self, layer_names, bool): """Set list of layers' trainablility. """ layer_names = to_list(layer_names) for layer_name in layer_names: row = self._find_tt_row_by_name(layer_name, self._trainable_table_) if row is None: raise Exception("Can not find layer_name!") else: index = row[0] self._trainable_table_[index][-1] = bool
def get_observe_backward_func(self, observe_nodes): if self.gt_nodes_ is None: raise Exception( "You must call set_gt_nodes method before call observe_backward method!" ) observe_nodes = to_list(observe_nodes) inputs = self.in_nodes_ + self.gt_nodes_ + [self.tr_phase_node_] timer = Timer() f_observe_backward = K.function_no_given(inputs, observe_nodes) timer.show("Compiling f_observe_backward time:") return f_observe_backward
def __init__(self, in_layers=[], out_layers=[], any_layers=[]): self._in_layers_ = to_list(in_layers) self._out_layers_ = to_list(out_layers) self._any_layers_ = to_list(any_layers) self._gt_nodes_ = None self._f_predict = None self._epoch_ = 0 self._iter_ = 0 self._tr_time_ = 0. self._tr_phase_node_ = K.common_tr_phase_node self._effective_layers_ = self.get_effective_layers( self._in_layers_, self._out_layers_) self._check_duplicate_name(self._effective_layers_) self._trainable_table_ = self._init_trainable_table( self._effective_layers_) self._f_optimize_ = None
def run_function(self, func, z, batch_size, tr_phase): """Return output of a function given value. Args: func: function z: ndarray | list of ndarray. Can be [inputs] for computing forward, or [inputs]+[outputs] for computing backward Returns: list of ndarray """ # Format data z = to_list(z) z = format_data_list(z) # Calculating all in same time if batch_size is None: in_list = z + [tr_phase] y_out = func(*in_list) # Calculating in batch else: N = len(z[0]) batch_num = int(np.ceil(float(N) / batch_size)) n_out_nodes = len(self.out_nodes_) y_out = [] # list of batch_y_out for i1 in xrange(batch_num): in_list = [ e[i1 * batch_size:min((i1 + 1) * batch_size, N)] for e in z ] + [tr_phase] batch_y_out = func(*in_list) # list of ndarray y_out.append(batch_y_out) def _reform(y_out): outs = [] for i1 in xrange(len(y_out[0])): tmp_list = [] for j1 in xrange(len(y_out)): tmp_list.append(y_out[j1][i1]) out = np.concatenate(tmp_list, axis=0) outs.append(out) return outs y_out = _reform(y_out) return y_out
def run_function(self, func, z, batch_size, tr_phase): """Return output of a function given value. Args: func: function z: ndarray | list of ndarray. Can be [inputs] for computing forward, or [inputs]+[outputs] for computing backward Returns: list of ndarray """ # Format data z = to_list(z) z = format_data_list(z) # Calculating all in same time if batch_size is None: in_list = z + [tr_phase] y_out = func(*in_list) # Calculating in batch else: N = len(z[0]) batch_num = int(np.ceil(float(N) / batch_size)) n_out_nodes = len(self.out_nodes_) y_out = [] # list of batch_y_out for i1 in xrange(batch_num): in_list = [e[i1*batch_size : min((i1+1)*batch_size, N)] for e in z] + [tr_phase] batch_y_out = func(*in_list) # list of ndarray y_out.append(batch_y_out) def _reform(y_out): outs = [] for i1 in xrange(len(y_out[0])): tmp_list = [] for j1 in xrange(len(y_out)): tmp_list.append(y_out[j1][i1]) out = np.concatenate(tmp_list, axis=0) outs.append(out) return outs y_out = _reform(y_out) return y_out
def add_models(self, mds): """Add list of model to existing model, update in_layers, out_layers, any_layers and trainable_table. """ mds = to_list(mds) new_in_layers = self.in_layers_ new_out_layers = self.out_layers_ new_any_layers = self.any_layers_ init_trainable_table = self.trainable_table_ for md in mds: new_in_layers += md.in_layers_ new_out_layers += md.out_layers_ new_any_layers += md.any_layers_ init_trainable_table += md.trainable_table_ self.__init__(new_in_layers, new_out_layers, new_any_layers) # New trainable_table is inherited from old concatenated trainable_table for row1 in init_trainable_table: row2 = self._find_tt_row_by_name(row1[1].name_, self._trainable_table_) if row2: index = row2[0] self._trainable_table_[index][-1] = row1[-1]
def predict(self, x, batch_size=100): # format data x = to_list(x) x = [K.format_data(e) for e in x] # compile predict model if not hasattr(self, '_f_predict'): self._f_predict = K.function_no_given(self._in_nodes_, self._tr_phase_node_, self._out_nodes_) # do predict # put all data in GPU if batch_size is None: in_list = x + [0.] y_out = self._f_predict(*in_list) # put batch data in GPU else: N = len(x[0]) batch_num = int(np.ceil(float(N) / batch_size)) n_out_nodes = len(self._out_nodes_) y_out = [[] for e in self._out_nodes_] for i1 in xrange(batch_num): in_list = [ e[i1 * batch_size:min((i1 + 1) * batch_size, N)] for e in x ] + [0.] batch_y_out = self._f_predict(*in_list) for j1 in xrange(n_out_nodes): y_out[j1].append(batch_y_out[j1]) # get y_out y_out = [np.concatenate(e, axis=0) for e in y_out] if len(y_out) == 1: return y_out[0] else: return y_out
def _evaluate(self, x, y, eval_type): # get metric losses node loss_nodes = [] for metric in self._metrics_: # if use default objective if type(metric) is str: assert len(self._md_.out_nodes_)==len(self._md_.gt_nodes_), "If you are using default objectives, " \ + "out_node of out_layers must match ground truth!" loss_node = sum([ obj.get(metric)(pred_node, gt_node) for pred_node, gt_node in zip(self._md_.out_nodes_, self._md_.gt_nodes_) ]) # if user define their objective function elif isfunction(metric): loss_node = metric(self._md_) else: loss_node = metric loss_nodes.append(loss_node) # compile evaluation function if not hasattr(self, '_f_evaluate'): print 'compiling evaluation function ..' inputs = self._md_.in_nodes_ + self._md_.gt_nodes_ + [ self._md_.tr_phase_node_ ] self._f_evaluate = K.function_no_given(inputs, loss_nodes) print 'compile finished. ' # calculate metric values t1 = time.time() if self._generator_: generator = self._generator_ else: generator = self._DefaultGenerator(self._batch_size_) n_all = 0. cnt = 0. metric_vals = np.zeros(len(self._metrics_)) batch_num = sum(1 for it in generator.generate(x, y)) for batch_x, batch_y in generator.generate(x, y): batch_x = to_list(batch_x) batch_y = to_list(batch_y) curr_batch_size = batch_x[0].shape[0] n_all += curr_batch_size cnt += 1. if self._transformer_: (batch_x, batch_y) = self._transformer_.transform(batch_x, batch_y) batch_x = format_data_list(batch_x) batch_y = format_data_list(batch_y) in_list = batch_x + batch_y + [0.] batch_metric_vals = np.array(self._f_evaluate(*in_list)) metric_vals += batch_metric_vals * curr_batch_size if self._verbose_ == 1: self._print_progress(batch_num, cnt) metric_vals /= n_all # timer t2 = time.time() # print results self._print_time_results(eval_type, self._metrics_, metric_vals, t2 - t1)
def fit(self, x, y, batch_size=100, n_epochs=10, loss_func='categorical_crossentropy', optimizer=SGD(lr=0.01, momentum=0.9), clip=None, callbacks=[], shuffle=True, transformer=None, verbose=1): """Fit data and train model. The data is reshuffled after every epoch. Args: x: ndarray | list of numpy ndarray. y: ndarray | list of numpy ndarray. batch_size: integar. n_epoch: integar. Number of training epochs. loss_func: str | function. optimizer: optimization object. clip: real value. callbacks: list of Callback object. shuffle: bool. transformer: Transformation object. verbose: 0 | 1 | 2 Returns: None. (All trained models, results should be saved using callbacks.) """ x = to_list(x) y = to_list(y) # Format data x = format_data_list(x) y = format_data_list(y) # Train memory usage print "Training", self._show_memory_usage(self._effective_layers_, batch_size) # Compile optimization function timer = Timer() target_dim_list = self._get_target_dim_list(x, y, transformer) f_optimize = self.get_optimization_func(target_dim_list, loss_func, optimizer, clip) timer.show("Compiling f_optimize time:") # Compile for callback timer = Timer() if callbacks is not None: callbacks = to_list(callbacks) for callback in callbacks: callback.compile(self) timer.show("Compiling callbacks time:") # Train N = len(x[0]) batch_num = int(np.ceil(float(N) / batch_size)) max_epoch = n_epochs + self.epoch_ # Callback print '\n', self.epoch_, 'th epoch:' for callback in callbacks: if (self.epoch_ % callback.call_freq_ == 0): callback.call() while self.epoch_ < max_epoch: # Shuffle data if shuffle: x, y = supports.shuffle(x, y) # Train t1 = time.time() loss_list = [] for i2 in xrange(batch_num): batch_x = [e[i2*batch_size : min((i2+1)*batch_size, N)] for e in x] batch_y = [e[i2*batch_size : min((i2+1)*batch_size, N)] for e in y] if transformer: (batch_x, batch_y) = transformer.transform(batch_x, batch_y) batch_x = format_data_list(batch_x) batch_y = format_data_list(batch_y) in_list = batch_x + batch_y + [1.] # training phase loss = f_optimize(*in_list)[0] loss_list.append(loss) # self._iter_ += 1 if verbose==1: self.print_progress(self.epoch_, batch_num, i2) if verbose==2: self.print_progress_loss(self.epoch_, batch_num, i2, loss) t2 = time.time() self._tr_time_ += (t2 - t1) if verbose!=0: print '\n', ' tr_time: ', "%.2f" % (t2-t1), 's' # print an empty line self._epoch_ += 1 # Callback for callback in callbacks: if (self.epoch_ % callback.call_freq_ == 0): callback.call()
def fit(self, x, y, batch_size=100, n_epochs=10, loss_func='categorical_crossentropy', optimizer=SGD(lr=0.01, rho=0.9), clip=None, callbacks=[], shuffle=True, verbose=1): x = to_list(x) y = to_list(y) # format x = [K.format_data(e) for e in x] y = [K.format_data(e) for e in y] # shuffle data if shuffle: x, y = supports.shuffle(x, y) # check data self._check_data(y, loss_func) # init gt_nodes self._gt_nodes_ = [K.placeholder(e.ndim) for e in y] # memory usage print "Train", self._show_memory_usage(self._layer_list_, batch_size) # default objective if type(loss_func) is str: assert len(self._out_nodes_)==len(self._gt_nodes_), "If you are using default objectives, " \ + "out_node of out_layers must match ground truth!" loss_node = sum([ obj.get(loss_func)(pred_node, gt_node) for pred_node, gt_node in zip( self._out_nodes_, self._gt_nodes_) ]) # user defined objective else: loss_node = loss_func(self._out_nodes_, self._any_nodes_, self._gt_nodes_) #loss_node = loss_func( self ) # gradient gparams = K.grad(loss_node + self._reg_value_, self._params_) # todo clip gradient if clip is not None: gparams = [K.clip(gparam, -clip, clip) for gparam in gparams] # gradient based opt param_updates = optimizer.get_updates(self._params_, gparams) # get all updates updates = param_updates + self._inner_updates_ # compile for callback if callbacks is not None: callbacks = to_list(callbacks) for callback in callbacks: callback.compile(self) # compile model input_nodes = self._in_nodes_ + self._gt_nodes_ output_nodes = [loss_node] f = K.function_no_given(input_nodes, self._tr_phase_node_, output_nodes, updates) # train N = len(x[0]) batch_num = int(np.ceil(float(N) / batch_size)) n_abs_epoch = n_epochs + self._epoch_ # callback print '\n0th epoch:' for callback in callbacks: if (self._epoch_ % callback.call_freq == 0): callback.call() while self._epoch_ < n_abs_epoch: self._epoch_ += 1 # train t1 = time.time() loss_list = [] for i2 in xrange(batch_num): batch_x = [ e[i2 * batch_size:min((i2 + 1) * batch_size, N)] for e in x ] batch_y = [ e[i2 * batch_size:min((i2 + 1) * batch_size, N)] for e in y ] in_list = batch_x + batch_y + [1.] loss = f(*in_list)[0] # training phase loss_list.append(loss) if verbose == 1: self._print_progress(self._epoch_, batch_num, i2) if verbose == 2: self._print_progress_loss(self._epoch_, batch_num, i2, loss) t2 = time.time() self._tr_time_ += (t2 - t1) if verbose != 0: print '\n', ' tr_time: ', "%.2f" % ( t2 - t1), 's' # print an empty line # callback for callback in callbacks: if (self._epoch_ % callback.call_freq == 0): callback.call()
def fit(self, x, y, batch_size=100, n_epochs=10, loss_func='categorical_crossentropy', optimizer=SGD(lr=0.01, momentum=0.9), clip=None, callbacks=[], shuffle=True, transformer=None, verbose=1): """Fit data and train model. The data is reshuffled after every epoch. Args: x: ndarray | list of numpy ndarray. y: ndarray | list of numpy ndarray. batch_size: integar. n_epoch: integar. Number of training epochs. loss_func: str | function. optimizer: optimization object. clip: real value. callbacks: list of Callback object. shuffle: bool. transformer: Transformation object. verbose: 0 | 1 | 2 Returns: None. (All trained models, results should be saved using callbacks.) """ x = to_list(x) y = to_list(y) # Format data x = format_data_list(x) y = format_data_list(y) # Train memory usage print "Training", self._show_memory_usage(self._effective_layers_, batch_size) # Compile optimization function timer = Timer() target_dim_list = self._get_target_dim_list(x, y, transformer) f_optimize = self.get_optimization_func(target_dim_list, loss_func, optimizer, clip) timer.show("Compiling f_optimize time:") # Compile for callback timer = Timer() if callbacks is not None: callbacks = to_list(callbacks) for callback in callbacks: callback.compile(self) timer.show("Compiling callbacks time:") # Train N = len(x[0]) batch_num = int(np.ceil(float(N) / batch_size)) max_epoch = n_epochs + self.epoch_ # Callback print '\n', self.epoch_, 'th epoch:' for callback in callbacks: if (self.epoch_ % callback.call_freq_ == 0): callback.call() while self.epoch_ < max_epoch: # Shuffle data if shuffle: x, y = supports.shuffle(x, y) # Train t1 = time.time() loss_list = [] for i2 in xrange(batch_num): batch_x = [ e[i2 * batch_size:min((i2 + 1) * batch_size, N)] for e in x ] batch_y = [ e[i2 * batch_size:min((i2 + 1) * batch_size, N)] for e in y ] if transformer: (batch_x, batch_y) = transformer.transform(batch_x, batch_y) batch_x = format_data_list(batch_x) batch_y = format_data_list(batch_y) in_list = batch_x + batch_y + [1.] # training phase loss = f_optimize(*in_list)[0] loss_list.append(loss) # self._iter_ += 1 if verbose == 1: self.print_progress(self.epoch_, batch_num, i2) if verbose == 2: self.print_progress_loss(self.epoch_, batch_num, i2, loss) t2 = time.time() self._tr_time_ += (t2 - t1) if verbose != 0: print '\n', ' tr_time: ', "%.2f" % ( t2 - t1), 's' # print an empty line self._epoch_ += 1 # Callback for callback in callbacks: if (self.epoch_ % callback.call_freq_ == 0): callback.call()
def train_on_batch(self, batch_x, batch_y, loss_func='categorical_crossentropy', optimizer=SGD(lr=0.01, momentum=0.9), clip=None, callbacks=[], transformer=None, recompile=False): """Train model on single batch data. Args: batch_x: ndarray | list of ndarray. batch_y: ndarray | list of ndarray. loss_func: str | function. optimizer: optimization object. clip: real value. callbacks: list of Callback object. transformer: Transformation object. recompile: bool. Recompile the optimize function or not. Returns: None. (All trained models, results should be saved using callbacks.) """ batch_x = to_list(batch_x) batch_y = to_list(batch_y) # Format data batch_x = format_data_list(batch_x) batch_y = format_data_list(batch_y) # Compile optimization function if (not self._f_optimize_) or (recompile): timer = Timer() target_dim_list = self._get_target_dim_list( batch_x, batch_y, transformer) self._f_optimize_ = self.get_optimization_func( target_dim_list, loss_func, optimizer, clip) timer.show("Compiling f_optimize time:") batch_size = len(batch_x[0]) self._show_memory_usage(self._effective_layers_, batch_size) # Compile for callback timer = Timer() if callbacks is not None: callbacks = to_list(callbacks) for callback in callbacks: callback.compile(self) timer.show("Compiling callbacks time:") for callback in callbacks: if (self.iter_ % callback.call_freq_ == 0): print self.iter_, 'th iteration:' callback.call() # Train t1 = time.time() if transformer: (batch_x, batch_y) = transformer.transform(batch_x, batch_y) batch_x = format_data_list(batch_x) batch_y = format_data_list(batch_y) in_list = batch_x + batch_y + [1.] # training phase loss = self._f_optimize_(*in_list)[0] self._iter_ += 1 t2 = time.time() self._tr_time_ += (t2 - t1) return loss