def _compute_over_subset(self, subset, inputs, targets, monitors_dict, monitor_function, monitors_outservice_dict, plot): inputs = raise_to_list(inputs) targets = raise_to_list(targets) if inputs is not None and len(monitors_dict) > 0: monitors = {key: [] for key in monitors_dict.keys()} data = [minibatch(input, self.batch_size, self.min_batch_size) for input in inputs] if targets is not None and not self.unsupervised: data += [minibatch(target, self.batch_size, self.min_batch_size) for target in targets] for batch in min_normalized_izip(*data): _outs = raise_to_list(monitor_function(*batch)) current_monitors = zip(monitors_dict.keys(), _outs) for name, val in current_monitors: val = numpy.asarray(val) monitors[name].append(val) # get the mean values for the batches current_mean_monitors = {key: numpy.mean(vals, 0) for key, vals in monitors.items()} # log the mean values! log.info('%s monitors: %s', subset, str(current_mean_monitors)) # send the values to their outservices for name, service in monitors_outservice_dict.items(): if name in current_mean_monitors and service: service.write(current_mean_monitors[name], subset) # if there is a plot, also send them over! if plot: plot.update_plots(epoch=self.epoch_counter, monitors=current_mean_monitors)
def _compute_over_subset(self, subset, inputs, targets, monitors_dict, monitor_function, monitors_outservice_dict, plot): inputs = raise_to_list(inputs) targets = raise_to_list(targets) if inputs is not None and len(monitors_dict) > 0: monitors = {key: [] for key in monitors_dict.keys()} data = [minibatch(input, self.batch_size, self.min_batch_size) for input in inputs] if targets is not None and not self.unsupervised: data += [minibatch(target, self.batch_size, self.min_batch_size) for target in targets] for batch in min_normalized_izip(*data): _outs = raise_to_list(monitor_function(*batch)) current_monitors = zip(monitors_dict.keys(), _outs) for name, val in current_monitors: val = numpy.asarray(val) monitors[name].append(val) # get the mean values for the batches current_mean_monitors = {key: numpy.mean(vals, 0) for key, vals in monitors.items()} # log the mean values! log.info('%s monitors: %s', subset, str(current_mean_monitors)) # send the values to their outservices for name, service in monitors_outservice_dict.items(): if name in current_mean_monitors and service: service.write(current_mean_monitors[name], subset) # if there is a plot, also send them over! if plot: plot.update_plots(epoch=self.epoch_counter, monitors=current_mean_monitors)
def test_minibatch(): """Test that we get real minibatches across the dataset""" import numpy from opendeep.utils.batch import minibatch dataset = TEDLIUMDataset() inputs = dataset.train_inputs first = iter(minibatch( inputs, batch_size=128 )).next() assert isinstance(first, numpy.ndarray), first assert first.shape == (128,dataset.window_size), first.shape
def _perform_one_epoch(self, f_learn, plot=None): """ Performs a single training iteration with the given learn function. """ self.epoch_counter += 1 t = time.time() log.info('EPOCH %s', str(self.epoch_counter)) # set the noise switches on for training function! (this is where things like dropout happen) if not self.model.switches_on: self.model.turn_on_switches() ######### # train # ######### train_costs = [] train_monitors = {key: [] for key in self.train_monitors_dict.keys()} train_data = [ minibatch(input_data, self.batch_size, self.min_batch_size) for input_data in raise_to_list(self.dataset.train_inputs) ] if self.dataset.train_targets is not None and not self.unsupervised: train_data += [ minibatch(target, self.batch_size, self.min_batch_size) for target in raise_to_list(self.dataset.train_targets) ] for batch in min_normalized_izip(*train_data): _outs = raise_to_list(f_learn(*batch)) train_costs.append(_outs[0]) # handle any user defined monitors if len(train_monitors) > 0: current_monitors = zip(self.train_monitors_dict.keys(), _outs[1:]) for name, val in current_monitors: val = numpy.asarray(val) train_monitors[name].append(val) # get the mean values for the batches mean_train = numpy.mean(train_costs, 0) current_mean_monitors = {key: numpy.mean(vals, 0) for key, vals in train_monitors.items()} # log the mean values! log.info('Train cost: %s', trunc(mean_train)) if len(current_mean_monitors) > 0: log.info('Train monitors: %s', str(current_mean_monitors)) # send the values to their outservices if self.train_outservice: self.train_outservice.write(mean_train, "train") for name, service in self.train_monitors_outservice_dict.items(): if name in current_mean_monitors and service: service.write(current_mean_monitors[name], "train") # if there is a plot, also send them over! if plot: current_mean_monitors.update({TRAIN_COST_KEY: mean_train}) plot.update_plots(epoch=self.epoch_counter, monitors=current_mean_monitors) # set the noise switches off for valid and test sets! we assume unseen data is noisy anyway :) if self.model.switches_on: self.model.turn_off_switches() ######### # valid # ######### self._compute_over_subset("valid", self.dataset.valid_inputs, self.dataset.valid_targets, self.valid_monitors_dict, self.valid_monitor_function, self.valid_monitors_outservice_dict, plot) ######## # test # ######## self._compute_over_subset("test", self.dataset.test_inputs, self.dataset.test_targets, self.test_monitors_dict, self.test_monitor_function, self.test_monitors_outservice_dict, plot) ########### # cleanup # ########### # check for early stopping on train costs cost = numpy.sum(train_costs) # if the cost improved, reset the patience and record the best cost. if cost < self.best_cost * self.early_stop_threshold: self.patience = 0 self.best_cost = cost # save the parameters that made it the best self.best_params = {key: param.get_value(borrow=False) for key, param in self.params.items()} elif not numpy.isnan(cost): self.patience += 1 # check for stopping either from n_epochs or from threshold/patience stop = False if self.epoch_counter >= self.n_epoch: log.info("Stopping (reached max number of epochs)...") stop = True if self.patience >= self.early_stop_length: log.info("Stopping early (reached stop threshold)...") stop = True timing = time.time() - t self.times.append(timing) log.info('time: ' + make_time_units_string(timing)) log.debug('remaining time: ' + make_time_units_string((self.n_epoch - self.epoch_counter) * numpy.mean(self.times))) if (self.epoch_counter % self.save_frequency) == 0: #save params self.model.save_params('trained_epoch_' + str(self.epoch_counter)) # ANNEAL! if not stop: # perform the appropriate decay on the decay functions/parameters for this optimizer and model for decay_param in self.get_decay_params(): decay_param.decay() # return whether or not to stop this epoch return stop
def _perform_one_epoch(self, f_learn, plot=None): """ Performs a single training iteration with the given learn function. """ self.epoch_counter += 1 t = time.time() log.info('EPOCH %s', str(self.epoch_counter)) # set the noise switches on for training function! (this is where things like dropout happen) if not self.model.switches_on: self.model.turn_on_switches() ######### # train # ######### train_costs = [] train_monitors = {key: [] for key in self.train_monitors_dict.keys()} train_data = [ minibatch(input_data, self.batch_size, self.min_batch_size) for input_data in raise_to_list(self.dataset.train_inputs) ] if self.dataset.train_targets is not None and not self.unsupervised: train_data += [ minibatch(target, self.batch_size, self.min_batch_size) for target in raise_to_list(self.dataset.train_targets) ] for batch in min_normalized_izip(*train_data): _outs = raise_to_list(f_learn(*batch)) train_costs.append(_outs[0]) # handle any user defined monitors (if different from the train cost) if len(train_monitors) > 0: current_monitors = zip(self.train_monitors_dict.keys(), _outs[1:]) for name, val in current_monitors: val = numpy.asarray(val) train_monitors[name].append(val) # get the mean values for the batches mean_train = numpy.mean(train_costs, 0) current_mean_monitors = {key: numpy.mean(vals, 0) for key, vals in train_monitors.items()} # log the mean values! log.info('Train cost: %s', trunc(mean_train)) if len(current_mean_monitors) > 0: log.info('Train monitors: %s', str(current_mean_monitors)) # send the values to their outservices for name, service in self.train_monitors_outservice_dict.items(): if name in current_mean_monitors and service: service.write(current_mean_monitors[name], "train") # if there is a plot, also send them over! if plot: plot.update_plots(epoch=self.epoch_counter, monitors=current_mean_monitors) # set the noise switches off for valid and test sets! we assume unseen data is noisy anyway :) if self.model.switches_on: self.model.turn_off_switches() ######### # valid # ######### self._compute_over_subset("valid", self.dataset.valid_inputs, self.dataset.valid_targets, self.valid_monitors_dict, self.valid_monitor_function, self.valid_monitors_outservice_dict, plot) ######## # test # ######## self._compute_over_subset("test", self.dataset.test_inputs, self.dataset.test_targets, self.test_monitors_dict, self.test_monitor_function, self.test_monitors_outservice_dict, plot) ########### # cleanup # ########### # check for early stopping on train costs cost = numpy.sum(train_costs) # if the cost improved, reset the patience and record the best cost. if cost < self.best_cost * self.early_stop_threshold: self.patience = 0 self.best_cost = cost # save the parameters that made it the best self.best_params = self.model.get_param_values(borrow=False) elif not numpy.isnan(cost): self.patience += 1 # check for stopping either from n_epochs or from threshold/patience stop = False if self.epoch_counter >= self.n_epoch: log.info("Stopping (reached max number of epochs)...") stop = True if self.patience >= self.early_stop_length: log.info("Stopping early (reached stop threshold)...") stop = True timing = time.time() - t self.times.append(timing) log.info('time: ' + make_time_units_string(timing)) log.debug('remaining time: ' + make_time_units_string((self.n_epoch - self.epoch_counter) * numpy.mean(self.times))) if (self.epoch_counter % self.save_frequency) == 0: #save params self.model.save_params('trained_epoch_' + str(self.epoch_counter)) # ANNEAL! if not stop: # perform the appropriate decay on the decay functions/parameters for this optimizer and model for decay_param in self.get_decay_params(): decay_param.decay() # return whether or not to stop this epoch return stop