def train_model(): m = build_model() stop = climin.stops.any_([ climin.stops.after_n_iterations(max_iter), ]) pause = climin.stops.modulo_n_iterations(n_report) weight_decay = ((m.parameters.hidden_to_out ** 2).sum()) # + (m.parameters.hidden_to_hidden_0**2).sum() # + (m.parameters.hidden_to_out**2).sum()) weight_decay /= m.exprs['inpt'].shape[0] m.exprs['true_loss'] = m.exprs['loss'] m.exprs['loss'] = m.exprs['loss'] + c_wd * weight_decay f_wd = m.function(['inpt'], c_wd * weight_decay) n_wrong = 1 - T.eq(T.argmax(m.exprs['output'], axis=1), T.argmax(m.exprs['target'], axis=1)).mean() f_n_wrong = m.function(['inpt', 'target'], n_wrong) losses = [] v_losses = [] print 'max iter', max_iter start = time.time() # Set up a nice printout. keys = '#', 'loss', 'val loss', 'seconds', 'wd', 'train emp', 'test emp' max_len = max(len(i) for i in keys) header = '\t'.join(i for i in keys) print header print '-' * len(header) f_loss = m.function(['inpt', 'target'], ['true_loss', 'loss']) for i, info in enumerate(m.powerfit((X, Z), (TX, TZ), stop, pause)): if info['n_iter'] % n_report != 0: continue passed = time.time() - start losses.append(info['loss']) v_losses.append(info['val_loss']) #img = tile_raster_images(fe.parameters['in_to_hidden'].T, image_dims, feature_dims, (1, 1)) #save_and_display(img, 'filters-%i.png' % i) info.update({ 'time': passed, 'l2-loss': scalar(f_wd(X)), 'train_emp': scalar(f_n_wrong(X, Z)), 'test_emp': scalar(f_n_wrong(TX, TZ)), }) row = '%(n_iter)i\t%(loss)g\t%(val_loss)g\t%(time)g\t%(l2-loss)g\t%(train_emp)g\t%(test_emp)g' % info print row np.savez_compressed(savepath, parameters=gp.as_numpy_array(m.parameters.data[...]))
def iter_fit(self, *fit_data): start = time.time() for info in self.model.iter_fit(*fit_data): if self.pause(info): # Take care of batch norm # Things done here shouldn't affect running metrics since no learning is supposed to happen. if self.using_bn: self.model.phase_select(phase_id='valid') if 'loss' not in info: info['loss'] = ma.scalar( self.score_fun(self.model.score, *self.data['train']) ) if self.evaluate: info['val_loss'] = ma.scalar( self.score_fun(self.model.score, *self.data['val']) ) if info['val_loss'] < self.best_loss: self.best_loss = info['val_loss'] self.best_pars = self.model.parameters.data.copy() self.losses.append((info['loss'], info['val_loss'])) else: self.losses.append(info['loss']) if self.test: info['test_avg'] = ma.scalar( self.score_fun(self.model.score, *self.data['test']) ) self.test_performance.append(info['test_avg']) self.runtime = time.time() - start info.update({ 'best_loss': self.best_loss, 'best_pars': self.best_pars, 'runtime': self.runtime }) self.n_epochs_done = info['n_iter'] / self.n_report self.n_iters_done = info['n_iter'] # Return to training mode, keep learning running metrics. if self.using_bn: self.model.phase_select(phase_id='train') yield info if self.stop(info): break
def iter_fit(self, *fit_data): start = time.time() for info in self.model.iter_fit(*fit_data): if self.pause(info): # Take care of batch norm # Things done here shouldn't affect running metrics since no learning is supposed to happen. if self.using_bn: self.model.phase_select(phase_id='valid') if 'loss' not in info: info['loss'] = ma.scalar( self.score_fun(self.model.score, *self.data['train'])) if self.evaluate: info['val_loss'] = ma.scalar( self.score_fun(self.model.score, *self.data['val'])) if info['val_loss'] < self.best_loss: self.best_loss = info['val_loss'] self.best_pars = self.model.parameters.data.copy() self.losses.append((info['loss'], info['val_loss'])) else: self.losses.append(info['loss']) if self.test: info['test_avg'] = ma.scalar( self.score_fun(self.model.score, *self.data['test'])) self.test_performance.append(info['test_avg']) self.runtime = time.time() - start info.update({ 'best_loss': self.best_loss, 'best_pars': self.best_pars, 'runtime': self.runtime }) self.n_epochs_done = info['n_iter'] / self.n_report self.n_iters_done = info['n_iter'] # Return to training mode, keep learning running metrics. if self.using_bn: self.model.phase_select(phase_id='train') yield info if self.stop(info): break
def iter_fit(self, *fit_data): """Iteratively fit the given training data. The values yielded from this function will be climin info dictionaries stripped from any numpy or gnumpy arrays. """ start = time.time() for info in self.model.iter_fit(*fit_data, info_opt=self.current_info): interrupt = self.interrupt(info) if self.pause(info) or interrupt: info['val_loss'] = ma.scalar( self.model._f_loss(self.model.parameters.data, *self.data[self.val_key])) for i in self.info_keys: info['{}_loss'.format(i)] = ma.scalar( self.model._f_loss( self.model.parameters.data, *self.data[i])) cur_val_loss = info['%s_loss' % self.val_key] if cur_val_loss < self.best_loss: self.best_loss = cur_val_loss self.best_pars = self.model.parameters.data.copy() self.runtime += time.time() - start info.update({ 'best_loss': self.best_loss, 'best_pars': self.best_pars, 'datetime': datetime.datetime.now(), 'runtime': self.runtime }) filtered_info = clear_info(info) self.infos.append(filtered_info) self.current_info = info yield filtered_info start = time.time() if self.stop(info): self.stopped = True break if interrupt: break
def __call__(self, f_score, *data): """"Return the score of the data.""" batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1) score = 0. seen_samples = 0. for batch in batches: this_samples = int(batch[0].shape[self.sample_dims[0]]) score += f_score(*batch) * this_samples seen_samples += this_samples return ma.scalar(score / seen_samples)
def __call__(self, f_score, *data): """"Return the score of the data.""" batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1) score = 0. seen_samples = 0. for batch in batches: this_samples = batch[0].shape[self.sample_dims[0]] score += f_score(*batch) * this_samples seen_samples += this_samples return ma.scalar(score / seen_samples)
def __call__(self, f_score, *data): batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1) score = 0. seen_samples = 0. for batch in batches: x = batch[0] z = batch[1] this_samples = int(x.shape[0]) score += f_score(x, z) * this_samples seen_samples += this_samples return ma.scalar(score / seen_samples)
def iter_fit(self, *fit_data): """Iteratively fit the given training data. Generator function containing the main logic of the Trainer object. The arguments are of variable length and have to match that of the ``model.iter_fit()`` and ultimately the used loss function of that model. Each iteration of the fitting constitutes of running the optimizer of the model until either interrupt or pause returns True. In both cases, the generator will yield to the user. Additionally: - If interrupt returns True, the generator will stop yielding values afterwards. - stop will be tested. If it is true it will stop yielding afterwards and additionally ``.stopped`` will be set to True afterwards. - ``best_pars`` and ``best_loss`` will be updated. The values yielded from this function will be climin info dictionaries stripped from any numpy or gnumpy arrays. """ for info in self.model.iter_fit(*fit_data, info_opt=self.current_info): interrupt = self.interrupt(info) if self.pause(info) or interrupt: info['val_loss'] = ma.scalar( self.score(*self.eval_data[self.val_key])) cur_val_loss = info['%s_loss' % self.val_key] if cur_val_loss < self.best_loss: self.best_loss = cur_val_loss self.best_pars = self.model.parameters.data.copy() info['best_loss'] = self.best_loss info['best_pars'] = self.best_pars info.update({ 'datetime': datetime.datetime.now(), }) filtered_info = clear_info(info) self.infos.append(filtered_info) self.current_info = info yield info if self.stop(info): self.stopped = True break if interrupt: break
def iter_fit(self, *fit_data): """Iteratively fit the given training data. Generator function containing the main logic of the Trainer object. The arguments are of variable length and have to match that of the ``model.iter_fit()`` and ultimately the used loss function of that model. Each iteration of the fitting constitutes of running the optimizer of the model until either interrupt or pause returns True. In both cases, the generator will yield to the user. Additionally: - If interrupt returns True, the generator will stop yielding values afterwards. - stop will be tested. If it is true it will stop yielding afterwards and additionally ``.stopped`` will be set to True afterwards. - ``best_pars`` and ``best_loss`` will be updated. The values yielded from this function will be climin info dictionaries stripped from any numpy or gnumpy arrays. """ for info in self.model.iter_fit(*fit_data, info_opt=self.current_info): interrupt = self.interrupt(info) if self.pause(info) or interrupt: info['val_loss'] = ma.scalar(self.score(*self.data[self.val_key])) cur_val_loss = info['%s_loss' % self.val_key] if cur_val_loss < self.best_loss: self.best_loss = cur_val_loss self.best_pars = self.model.parameters.data.copy() info['best_loss'] = self.best_loss info['best_pars'] = self.best_pars info.update({ 'datetime': datetime.datetime.now(), }) filtered_info = clear_info(info) self.infos.append(filtered_info) self.current_info = info yield info if self.stop(info): self.stopped = True break if interrupt: break
def __call__(self, predict_f, *data): batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1) seen_samples = 0. score = 0. for batch in batches: x, z = batch y = predict_f(x) this_samples = int(y.shape[1]) errs = (y.argmax(axis=2) != z.argmax(axis=2)).sum() score += errs seen_samples += this_samples return ma.scalar(score / seen_samples)
def handle_update(self, fit_data): update_losses = {} for key, data in self.eval_data.items(): update_losses['%s_loss' % key] = ma.scalar(self.score(*data)) return update_losses
def powerfit(self, fit_data, eval_data, stop, report, eval_train_loss=True): """Iteratively fit the model. This is a convenience function which combines iteratively fitting a model with stopping criterions and keeping track of the best parameters found so far. An iterator of dictionaries is returned; values are only yielded in the case that the call `report(info)` returns True. The iterator stops as soon as the call `stop(info)` returns True. Each dictionary yielded is directly obtained from the optimizer used to optimize the loss. It is augmented with the keys `loss`, `best_pars` and `best_loss`. The best loss is obtained by evaluating the loss of the model (given by model.exprs['loss']) on `eval_data`, while training is done on `fit_data`. This method respects a ``true_loss`` entry in the ``exprs`` dictionary: if it is present, it will be used for reporting the loss and for comparing models throughout optimization instead of ``loss``, which will be used for the optimization itself. This makes it possible to add regularization terms to the loss and use other losses (such as the zero-one loss) for comparison of parameters. :param fit_data: A tuple containing arrays representing the data the model should be fitted on. :param eval_data: A tuple containing arrays representing the data the model should be evaluated on, and which gives which model is "best". :param stop: A function receiving an info dictionary which returns True if the iterator should stop. :param report: A function receiving an info dictionary which should return True if the iterator should yield a value. :returns: An iterator over info dictionaries. """ # TODO document eval train loss self.CTRL_C_FLAG = False signal.signal(signal.SIGINT, self._ctrl_c_handler) best_pars = None best_loss = float('inf') for info in self.iter_fit(*fit_data): if report(info): if 'loss' not in info: # Not all optimizers, e.g. ilne and gd, do actually # calculate the loss. if eval_train_loss: info['loss'] = ma.scalar(self.score(*fit_data)) else: info['loss'] = 0. info['val_loss'] = ma.scalar(self.score(*eval_data)) if info['val_loss'] < best_loss: best_loss = info['val_loss'] best_pars = self.parameters.data.copy() info['best_loss'] = best_loss info['best_pars'] = best_pars yield info if stop(info) or self.CTRL_C_FLAG: break
report_fun = report.OneLinePrinter( ['n_iter', 'runtime', 'loss', 'val_loss', 'test_avg'], spaces=['4', '7.4f', '5.4f', '7.4f', '7.4f'] ) score_fun = ash.MinibatchScoreFCN(max_samples=batch_size, sample_dims=[0, 0]) data = { 'train':(train_x, train_y), 'val':(valid_x, valid_y), 'test':(test_x, test_y) } test_fun = ash.MinibatchTestFCN(max_samples=batch_size, sample_dims=[0, 0]) initial_err = ma.scalar(score_fun(pkchu.score, *data['train'])) print 'Initial train loss: %.4f' % initial_err coach = PocketTrainer( model=pkchu, data=data, stop=stop, pause=pause, score_fun=score_fun, report_fun=report_fun, test_fun=test_fun, evaluate=True, test=True ) coach.fit() pkchu.parameters.data[...] = coach.best_pars plt.plot(coach.losses)
def RNN_EEG(n_neurons=100, batch_size=50, participant=[1], series=[1, 2, 3, 4, 5, 6, 7, 8, 9], subsample=0, imp_weights_skip=150, n_layers=1): format_string = '%d_%d_[' for p in participant: format_string += '%d,' % p format_string = format_string[:-1] format_string += ']_[' for s in series: format_string += '%d,' % s format_string = format_string[:-1] format_string += ']_%d_%d_%d_%d' net_info = format_string % (n_neurons, batch_size, subsample, imp_weights_skip, n_layers, time.time()) logging.info('--------------------------------') logging.info('EEG - DATA') logging.info( 'Rnn: n_neurons: %i, batch_size: %i, participant: %s, series: %s, subsample: %i, imp_weights_skip: %i' % (n_neurons, batch_size, participant, series, subsample, imp_weights_skip)) #optimizer = 'rmsprop', {'step_rate': 0.0001, 'momentum': 0.9, 'decay': 0.9} decay = 0.9 offset = 1e-6 mom = .9 step_rate = .1 optimizer = 'adadelta', { 'decay': decay, 'offset': offset, 'momentum': mom, 'step_rate': step_rate } # optimizer = 'adam' n_hiddens = [n_neurons] * n_layers logging.info('optimizer: %s' % str(optimizer)) m = SupervisedRnn(st.N_EEG_SENSORS, n_hiddens, st.N_EEG_TARGETS, out_transfer='sigmoid', loss='bern_ces', hidden_transfers=['tanh'] * n_layers, batch_size=batch_size, imp_weight=True, optimizer=optimizer) sX, sZ, sVX, sVZ, TX, TZ, seqlength, eventNames = get_shaped_input( participant, series, subsample) W = np.ones_like(sZ) WV = np.ones_like(sVZ) WT = np.ones_like(TX) W[:imp_weights_skip, :, :] = 0 WV[:imp_weights_skip, :, :] = 0 WT[:imp_weights_skip, :, :] = 0 m.exprs['true_loss'] = m.exprs['loss'] # TODO: Test Loss: Doesn't work, don't know why, trying a hacky workaround in test_loss() - don't know if right this way # f_loss = m.function(['inpt', 'target', 'imp_weight'], 'true_loss') # print(f_loss([TX[:, :, :], TZ[:, :, :]])) # print(m.score(TX[:,0:1,:], TZ[:,0:1,:], WT[:,0:1,:])) # similar error... # bern_ces changes the prediction!!! *facepalm* # ... just be careful to call it with a copy of the array def test_loss(): return bern_ces( m.predict(TX)[:imp_weights_skip, :seqlength, :], np.copy(TZ[:imp_weights_skip, :seqlength, :])).eval().mean() ''' def test_nll(): nll = 0 n_time_steps = 0 for x, z in zip(tx, tz): nll += f_loss(x[:, np.newaxis], z[:, np.newaxis]) * x.shape[0] n_time_steps += x.shape[0] return nll / n_time_steps ''' climin.initialize.randomize_normal(m.parameters.data, 0, 0.1) #climin.initialize.bound_spectral_radius(m.parameters.data) def plot(test_sample=0, save_name='images/%s_EEG.png' % net_info, test_loss=None): colors = ['blue', 'red', 'green', 'cyan', 'magenta'] figure, (axes) = plt.subplots(3, 1) #input_for_plot = sVX.transpose(1,0,2).reshape((-1, seqlength, st.N_EEG_SENSORS)).transpose(1,0,2)[:, 0:1, :] #target_for_plot = sVZ.transpose(1,0,2).reshape((-1, seqlength, st.N_EEG_TARGETS)).transpose(1,0,2)[:, 0:1, :] input_for_plot = TX[:, test_sample:test_sample + 1, :] # to be able to plot the test samples in correct length we have to determine where the '0' padding starts sample_length_arr = np.where( input_for_plot == np.zeros((st.N_EEG_SENSORS)))[0] if len(sample_length_arr != 0): sample_length = min( np.where( np.bincount(sample_length_arr) == st.N_EEG_SENSORS)[0]) else: sample_length = input_for_plot.shape[0] input_for_plot = input_for_plot[:sample_length] target_for_plot = TZ[:sample_length, test_sample:test_sample + 1, :] result = m.predict(input_for_plot) x_axis = np.arange(input_for_plot.shape[0]) for i in range(st.N_EEG_TARGETS): axes[0].set_title('TARGETS') axes[0].fill_between(x_axis, 0, target_for_plot[:, 0, i], facecolor=colors[i], alpha=0.8, label=eventNames[st.SEQ_EEG_TARGETS.index(i)]) #axes[0].plot(x_axis, target_for_plot[:, 0, i]) if test_loss: axes[1].set_title('RNN (overall test loss: %f)' % (test_loss)) else: axes[1].set_title('RNN') axes[1].plot(x_axis, result[:, 0, i], color=colors[i]) train_loss = [] val_loss = [] test_loss = [] for i in infos: train_loss.append(i['loss']) val_loss.append(i['val_loss']) test_loss.append(i['test_loss']) axes[2].set_title('LOSSES') axes[2].plot(np.arange(len(infos)), train_loss, label='train loss') axes[2].plot(np.arange(len(infos)), val_loss, label='validation loss') axes[2].plot(np.arange(len(infos)), test_loss, label='test loss') axes[0].legend( loc=0, shadow=True, fontsize='x-small') # loc: 0=best, 1=upper right, 2=upper left axes[2].legend(loc=0, shadow=True, fontsize='x-small') figure.subplots_adjust(hspace=0.5) figure.savefig(save_name) plt.close(figure) max_passes = 30 max_minutes = 200 max_iter = max_passes * sX.shape[1] / m.batch_size batches_per_pass = int(math.ceil(float(sX.shape[1]) / m.batch_size)) pause = climin.stops.ModuloNIterations( batches_per_pass * 1) # after each pass through all data stop = climin.stops.Any([ # climin.stops.TimeElapsed(max_minutes * 60), # maximal time in seconds climin.stops.AfterNIterations(max_iter), # maximal iterations # climin.stops.Patience('val_loss', batches_per_pass*10, grow_factor=1.5, threshold=0.0001), # kind of early stopping # climin.stops.NotBetterThanAfter(30, 100), # error under 30 after 100 iterations? ]) start = time.time() header = '#', 'seconds', 'loss', 'val loss', 'test loss' print '\t'.join(header) logging.info('\t'.join(header)) infos = [] try: os.makedirs('losses') except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and os.path.isdir('losses'): pass else: raise f = open('losses/%s_EEG.csv' % net_info, 'wt') try: writer = csv.writer(f) writer.writerow(('Train loss', 'Validation loss', 'Test Loss')) for i, info in enumerate( m.powerfit((sX, sZ, W), (sVX, sVZ, WV), stop=stop, report=pause, eval_train_loss=True)): info['loss'] = float(info['loss']) info['val_loss'] = float(info['val_loss']) info['test_loss'] = float(ma.scalar(test_loss())) writer.writerow( (info['loss'], info['val_loss'], info['test_loss'])) info.update({ 'time': time.time() - start, # 'spectral_radius': get_spectral_radius(m.parameters['recurrent_0']), }) template = '\t'.join([ '%(n_iter)i', '%(time)g', '%(loss)g', '%(val_loss)g', '%(test_loss)g' ]) row = template % info print row logging.info(row) filtered_info = dict( (k, v) for k, v in info.items() # if (not isinstance(v, (np.ndarray, gp.garray)) or v.size <= 1) and k not in ('args', 'kwargs')) if (not isinstance(v, (np.ndarray, )) or v.size <= 1) and k not in ('args', 'kwargs')) for key in filtered_info: if isinstance(filtered_info[key], np.float32): filtered_info[key] = float(filtered_info[key]) infos.append(filtered_info) finally: f.close() m.parameters.data[...] = info['best_pars'] save_timestmp = toUTCtimestamp(datetime.utcnow()) dot_idx = str(save_timestmp).index('.') save_timestmp = str(save_timestmp)[:dot_idx] logging.info('saved at: %s' % save_timestmp) plot(0, 'images/%s_eeg_test0.png' % net_info, test_loss()) plot(1, 'images/%s_eeg_test1.png' % net_info, test_loss()) plot(2, 'images/%s_eeg_test2.png' % net_info, test_loss()) plot(3, 'images/%s_eeg_test3.png' % net_info, test_loss())
def test_RNN(n_neurons=100, batch_size=50, participant=[1], series=[1, 2], subsample=10, imp_weights_skip=150, n_layers=1): format_string = '%d_%d_[' for p in participant: format_string += '%d,' % p format_string = format_string[:-1] format_string += ']_[' for s in series: format_string += '%d,' % s format_string = format_string[:-1] format_string += ']_%d_%d_%d_%d' net_info = format_string % (n_neurons, batch_size, subsample, imp_weights_skip, n_layers, time.time()) logging.info('--------------------------------') logging.info('Rnn: n_neurons: %i, batch_size: %i, participant: %s, series: %s, subsample: %i, imp_weights_skip: %i' % (n_neurons, batch_size, participant, series, subsample, imp_weights_skip)) #optimizer = 'rmsprop', {'step_rate': 0.0001, 'momentum': 0.9, 'decay': 0.9} decay = 0.9 offset = 1e-6 mom = .9 step_rate = .1 optimizer = 'adadelta', {'decay': decay, 'offset': offset, 'momentum': mom, 'step_rate': step_rate} # optimizer = 'adam' n_hiddens = [n_neurons] * n_layers logging.info('optimizer: %s' % str(optimizer)) m = SupervisedRnn( st.N_EMG_SENSORS, n_hiddens, st.N_EMG_TARGETS, out_transfer='sigmoid', loss='bern_ces', hidden_transfers=['tanh'] * n_layers, batch_size=batch_size, imp_weight=True, optimizer=optimizer) sX, sZ, sVX, sVZ, sTX, sTZ, eventNames = get_shaped_input(participant, series, subsample) W = np.ones_like(sZ) WV = np.ones_like(sVZ) #WT = np.ones_like(TX) W[:imp_weights_skip, :, :] = 0 WV[:imp_weights_skip, :, :] = 0 #WT[:imp_weights_skip, :, :] = 0 m.exprs['true_loss'] = m.exprs['loss'] # TODO: Test Loss: Doesn't work, don't know why, trying a hacky workaround in test_loss() - don't know if right this way # f_loss = m.function(['inpt', 'target', 'imp_weight'], 'true_loss') # print(f_loss([TX[:, :, :], TZ[:, :, :]])) # print(m.score(TX[:,0:1,:], TZ[:,0:1,:], WT[:,0:1,:])) # similar error... # bern_ces changes the prediction!!! *facepalm* # ... just be careful to call it with a copy of the array def test_loss(): pred = m.predict(sTX) return bern_ces(np.copy(sTZ), pred).eval().mean() climin.initialize.randomize_normal(m.parameters.data, 0, 0.1) def plot(test_sample=0, save_name='images/%s_test.png' % net_info, test_loss=None): colors = ['blue', 'red', 'green', 'cyan', 'magenta'] plt.figure(figsize=(40, 10)) figure, (axes) = plt.subplots(3, 1,figsize=(20,5)) input_for_plot = sTX target_for_plot = sTZ result = m.predict(input_for_plot) x_axis = np.arange(input_for_plot.shape[0]) for i in range(st.N_EMG_TARGETS): axes[0].set_title('TARGETS') axes[0].fill_between(x_axis, 0, target_for_plot[:, 0, i], facecolor=colors[i], alpha=0.8, label=eventNames[st.SEQ_EMG_TARGETS.index(i)]) if test_loss: axes[1].set_title('RNN (overall test loss: %f)' %(test_loss)) else: axes[1].set_title('RNN') axes[1].plot(x_axis, result[:, 0, i], color=colors[i]) train_loss = [] val_loss = [] test_loss = [] for i in infos: train_loss.append(i['loss']) val_loss.append(i['val_loss']) test_loss.append(i['test_loss']) axes[2].set_title('LOSSES') axes[2].plot(np.arange(len(infos)), train_loss, label='train loss') axes[2].plot(np.arange(len(infos)), val_loss, label='validation loss') axes[2].plot(np.arange(len(infos)), test_loss, label='test loss') axes[0].legend(loc=0, shadow=True, fontsize='x-small') # loc: 0=best, 1=upper right, 2=upper left axes[2].legend(loc=0, shadow=True, fontsize='x-small') figure.subplots_adjust(hspace=0.5) figure.savefig(save_name) plt.close(figure) max_passes = 400 max_minutes = 60 max_iter = max_passes * sX.shape[1] / m.batch_size batches_per_pass = int(math.ceil(float(sX.shape[1]) / m.batch_size)) pause = climin.stops.ModuloNIterations(batches_per_pass * 1) # after each pass through all data stop = climin.stops.Any([ climin.stops.TimeElapsed(max_minutes * 60), # maximal time in seconds climin.stops.AfterNIterations(max_iter) # maximal iterations #climin.stops.Patience('val_loss', batches_per_pass*50, grow_factor=2.0, threshold=0.00001), # kind of early stopping # climin.stops.NotBetterThanAfter(30, 100), # error under 30 after 100 iterations? ]) start = time.time() header = '#', 'seconds', 'loss', 'val loss', 'test loss' print '\t'.join(header) logging.info('\t'.join(header)) infos = [] try: os.makedirs('losses') except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and os.path.isdir('losses'): pass else: raise f = open('losses/%s.csv' % net_info, 'wt') try: writer = csv.writer(f) writer.writerow(('Train loss', 'Validation loss', 'Test Loss')) for i, info in enumerate(m.powerfit((sX, sZ, W), (sVX, sVZ, WV), stop=stop, report=pause, eval_train_loss=True)): info['loss'] = float(info['loss']) info['val_loss'] = float(info['val_loss']) info['test_loss'] = float(ma.scalar(test_loss())) writer.writerow((info['loss'], info['val_loss'], info['test_loss'])) info.update({ 'time': time.time() - start, }) template = '\t'.join( ['%(n_iter)i', '%(time)g', '%(loss)g', '%(val_loss)g', '%(test_loss)g']) row = template % info print row logging.info(row) filtered_info = dict( (k, v) for k, v in info.items() # if (not isinstance(v, (np.ndarray, gp.garray)) or v.size <= 1) and k not in ('args', 'kwargs')) if (not isinstance(v, (np.ndarray,)) or v.size <= 1) and k not in ('args', 'kwargs')) for key in filtered_info: if isinstance(filtered_info[key], np.float32): filtered_info[key] = float(filtered_info[key]) infos.append(filtered_info) finally: f.close() m.parameters.data[...] = info['best_pars'] save_timestmp = toUTCtimestamp(datetime.utcnow()) dot_idx = str(save_timestmp).index('.') save_timestmp = str(save_timestmp)[:dot_idx] logging.info('saved at: %s' % save_timestmp) plot(0, 'images/%s_emg_test0.png' % net_info, test_loss())
pkchu.parameters.data[...] = pickle.load(f) report_fun = report.OneLinePrinter( ['n_iter', 'runtime', 'loss', 'val_loss', 'test_avg'], spaces=['4', '7.4f', '5.4f', '7.4f', '7.4f']) score_fun = ash.MinibatchScoreFCN(max_samples=batch_size, sample_dims=[0, 0]) data = { 'train': (train_x, train_y), 'val': (valid_x, valid_y), 'test': (test_x, test_y) } test_fun = ash.MinibatchTestFCN(max_samples=batch_size, sample_dims=[0, 0]) initial_err = ma.scalar(score_fun(pkchu.score, *data['train'])) print 'Initial train loss: %.4f' % initial_err coach = PocketTrainer(model=pkchu, data=data, stop=stop, pause=pause, score_fun=score_fun, report_fun=report_fun, test_fun=test_fun, evaluate=True, test=True) coach.fit()