def _log_sample_answers(x_test, nn_model, mode, is_reverse_model): _logger.info('Model: {}'.format(get_model_full_path(is_reverse_model))) _logger.info('Start predicting responses of length {out_len} for {n_samples} samples with mode {mode}'.format( out_len=MAX_PREDICTIONS_LENGTH, n_samples=x_test.shape[0], mode=mode)) questions = transform_context_token_ids_to_sentences(x_test, nn_model.index_to_token) responses = get_nn_responses(x_test, nn_model, mode, output_candidates_num=LOG_CANDIDATES_NUM) _logger.info('Finished predicting! Logging...') for i, (question_ids, question) in enumerate(zip(x_test, questions)): laconic_logger.info('') # for better readability for j, response in enumerate(responses[i]): laconic_logger.info('%-35s\t --#=%02d--> \t%s' % (question, j + 1, response))
def _log_train_statistics(self): total_time = time.time() - self._training_start_time progress = self._cur_batch_id / self._batches_num # may be more than 100% if epochs num is more than 1 avr_time_per_batch = total_time / (self._cur_batch_id + 1) expected_time_per_epoch = avr_time_per_batch * self._batches_num total_training_time_in_percent = self._total_training_time / total_time batches_per_sec = self._cur_batch_id / total_time samples_per_sec = self._cur_batch_id * self._batch_size / total_time self._logger.info('Train statistics:\n') laconic_logger.info('batch:\t{batch_id} / {batches_num} ({progress:.1%})'.format( batch_id=self._cur_batch_id, batches_num=self._batches_num, progress=progress)) laconic_logger.info('loss:\t{loss:.2f}'.format(loss=self._cur_loss)) laconic_logger.info('time:\tepoch estimate {epoch_time:.2f} h | total {total_time:.2f} h | ' 'train {training_time:.2f} h ({training_time_percent:.1%})'.format( epoch_time=expected_time_per_epoch / 3600, # in hours total_time=total_time / 3600, # in hours training_time=self._total_training_time / 3600, # in hours training_time_percent=total_training_time_in_percent)) laconic_logger.info('speed:\t{batches_per_hour:.0f} batches/h, {samples_per_sec:.0f} samples/sec\n'.format( batches_per_hour=batches_per_sec * 3600, samples_per_sec=samples_per_sec))
def print_matrices_weights(self): laconic_logger.info('Net matrices weights:') params = get_all_params(self._net['dist']) values = get_all_param_values(self._net['dist']) total_network_size = 0 for p, v in zip(params, values): param_size = float(v.nbytes) / 1024 / 1024 laconic_logger.info( '\t{0:<40} dtype: {1:<10} shape: {2:<12} size: {3:<.2f}M'. format(p.name, v.dtype, v.shape, param_size)) total_network_size += param_size laconic_logger.info( 'Total network size: {0:.1f} Mb'.format(total_network_size))
def print_matrices_weights(self): laconic_logger.info('Net matrices weights:') params = get_all_params(self._net['dist']) values = get_all_param_values(self._net['dist']) total_network_size = 0 for p, v in zip(params, values): param_size = float(v.nbytes) / 1024 / 1024 # Work around numpy/python 3 regression: # http://www.markhneedham.com/blog/2017/11/19/python-3-typeerror-unsupported-format-string-passed-to-numpy-ndarray-__format__/ laconic_logger.info('\t{0:<40} dtype: {1:<10} shape: {2:<12} size: {3:<.2f}M'.format( p.name, repr(v.dtype), repr(v.shape), param_size)) total_network_size += param_size laconic_logger.info('Total network size: {0:.1f} Mb'.format(total_network_size))
def _log_sample_answers(self): self._logger.info('Sample responses for {} mode:'.format(self._prediction_mode_for_tests)) responses = get_nn_responses( self._val_contexts_tokens_ids, self._model, self._prediction_mode_for_tests, output_candidates_num=self._log_candidates_num) for context, response_candidates in zip(self._val_contexts, responses): laconic_logger.info('') # for better readability for i, response in enumerate(response_candidates): laconic_logger.info('{0: <35}\t #{1: <2d} --> \t{2}'.format(context, i + 1, response)) laconic_logger.info('') # for better readability
def load_weights(self): _logger.info('\nLoading saved weights from file:\n{}\n'.format( self.model_load_path)) saved_var_name_to_var = OrderedDict(np.load(self.model_load_path)) var_name_to_var = OrderedDict([ (v.name, v) for v in get_all_params(self._net['dist']) ]) initialized_vars, missing_vars, mismatched_vars = [], [], [] for var_name, var in iteritems(var_name_to_var): if var_name not in saved_var_name_to_var: missing_vars.append(var_name) continue default_var_value = var.get_value() saved_var_value = saved_var_name_to_var[var_name] if default_var_value.shape != saved_var_value.shape: mismatched_vars.append( (var_name, default_var_value.shape, saved_var_value.shape)) continue # Checks passed, set parameter value var.set_value(saved_var_value) initialized_vars.append(var_name) del saved_var_name_to_var[var_name] laconic_logger.info('\nRestored saved params:') for var_name in initialized_vars: laconic_logger.info('\t' + var_name) laconic_logger.warning('\nMissing saved params:') for var_name in missing_vars: laconic_logger.warning('\t' + var_name) laconic_logger.warning( '\nShapes-mismatched params (saved -> current):') for var_name, default_shape, saved_shape in mismatched_vars: laconic_logger.warning('\t{0:<40} {1:<12} -> {2:<12}'.format( var_name, saved_shape, default_shape)) laconic_logger.warning('\nUnused saved params:') for var_name in saved_var_name_to_var: laconic_logger.warning('\t' + var_name) laconic_logger.info('')
def print_layer_shapes(self): laconic_logger.info('Net shapes:') layers = get_all_layers(self._net['dist']) for l in layers: laconic_logger.info('\t%-20s \t%s' % (l.name, get_output_shape(l)))