def _log_sample_answers(x_test, nn_model, mode, is_reverse_model):
_logger.info('Model: {}'.format(get_model_full_path(is_reverse_model)))
_logger.info('Start predicting responses of length {out_len} for {n_samples} samples with mode {mode}'.format(
out_len=MAX_PREDICTIONS_LENGTH, n_samples=x_test.shape[0], mode=mode))
questions = transform_context_token_ids_to_sentences(x_test, nn_model.index_to_token)
responses = get_nn_responses(x_test, nn_model, mode, output_candidates_num=LOG_CANDIDATES_NUM)
_logger.info('Finished predicting! Logging...')
for i, (question_ids, question) in enumerate(zip(x_test, questions)):
laconic_logger.info('') # for better readability
for j, response in enumerate(responses[i]):
laconic_logger.info('%-35s\t --#=%02d--> \t%s' % (question, j + 1, response))
def _log_train_statistics(self):
total_time = time.time() - self._training_start_time
progress = self._cur_batch_id / self._batches_num # may be more than 100% if epochs num is more than 1
avr_time_per_batch = total_time / (self._cur_batch_id + 1)
expected_time_per_epoch = avr_time_per_batch * self._batches_num
total_training_time_in_percent = self._total_training_time / total_time
batches_per_sec = self._cur_batch_id / total_time
samples_per_sec = self._cur_batch_id * self._batch_size / total_time
self._logger.info('Train statistics:\n')
laconic_logger.info('batch:\t{batch_id} / {batches_num} ({progress:.1%})'.format(
batch_id=self._cur_batch_id, batches_num=self._batches_num, progress=progress))
laconic_logger.info('loss:\t{loss:.2f}'.format(loss=self._cur_loss))
laconic_logger.info('time:\tepoch estimate {epoch_time:.2f} h | total {total_time:.2f} h | '
'train {training_time:.2f} h ({training_time_percent:.1%})'.format(
epoch_time=expected_time_per_epoch / 3600, # in hours
total_time=total_time / 3600, # in hours
training_time=self._total_training_time / 3600, # in hours
training_time_percent=total_training_time_in_percent))
laconic_logger.info('speed:\t{batches_per_hour:.0f} batches/h, {samples_per_sec:.0f} samples/sec\n'.format(
batches_per_hour=batches_per_sec * 3600, samples_per_sec=samples_per_sec))
def print_matrices_weights(self):
laconic_logger.info('Net matrices weights:')
params = get_all_params(self._net['dist'])
values = get_all_param_values(self._net['dist'])
total_network_size = 0
for p, v in zip(params, values):
param_size = float(v.nbytes) / 1024 / 1024
laconic_logger.info(
'\t{0:<40} dtype: {1:<10} shape: {2:<12} size: {3:<.2f}M'.
format(p.name, v.dtype, v.shape, param_size))
total_network_size += param_size
laconic_logger.info(
'Total network size: {0:.1f} Mb'.format(total_network_size))
def print_matrices_weights(self):
laconic_logger.info('Net matrices weights:')
params = get_all_params(self._net['dist'])
values = get_all_param_values(self._net['dist'])
total_network_size = 0
for p, v in zip(params, values):
param_size = float(v.nbytes) / 1024 / 1024
# Work around numpy/python 3 regression:
# http://www.markhneedham.com/blog/2017/11/19/python-3-typeerror-unsupported-format-string-passed-to-numpy-ndarray-__format__/
laconic_logger.info('\t{0:<40} dtype: {1:<10} shape: {2:<12} size: {3:<.2f}M'.format(
p.name, repr(v.dtype), repr(v.shape), param_size))
total_network_size += param_size
laconic_logger.info('Total network size: {0:.1f} Mb'.format(total_network_size))
def _log_sample_answers(self):
self._logger.info('Sample responses for {} mode:'.format(self._prediction_mode_for_tests))
responses = get_nn_responses(
self._val_contexts_tokens_ids,
self._model,
self._prediction_mode_for_tests,
output_candidates_num=self._log_candidates_num)
for context, response_candidates in zip(self._val_contexts, responses):
laconic_logger.info('') # for better readability
for i, response in enumerate(response_candidates):
laconic_logger.info('{0: <35}\t #{1: <2d} --> \t{2}'.format(context, i + 1, response))
laconic_logger.info('') # for better readability
def load_weights(self):
_logger.info('\nLoading saved weights from file:\n{}\n'.format(
self.model_load_path))
saved_var_name_to_var = OrderedDict(np.load(self.model_load_path))
var_name_to_var = OrderedDict([
(v.name, v) for v in get_all_params(self._net['dist'])
])
initialized_vars, missing_vars, mismatched_vars = [], [], []
for var_name, var in iteritems(var_name_to_var):
if var_name not in saved_var_name_to_var:
missing_vars.append(var_name)
continue
default_var_value = var.get_value()
saved_var_value = saved_var_name_to_var[var_name]
if default_var_value.shape != saved_var_value.shape:
mismatched_vars.append(
(var_name, default_var_value.shape, saved_var_value.shape))
continue
# Checks passed, set parameter value
var.set_value(saved_var_value)
initialized_vars.append(var_name)
del saved_var_name_to_var[var_name]
laconic_logger.info('\nRestored saved params:')
for var_name in initialized_vars:
laconic_logger.info('\t' + var_name)
laconic_logger.warning('\nMissing saved params:')
for var_name in missing_vars:
laconic_logger.warning('\t' + var_name)
laconic_logger.warning(
'\nShapes-mismatched params (saved -> current):')
for var_name, default_shape, saved_shape in mismatched_vars:
laconic_logger.warning('\t{0:<40} {1:<12} -> {2:<12}'.format(
var_name, saved_shape, default_shape))
laconic_logger.warning('\nUnused saved params:')
for var_name in saved_var_name_to_var:
laconic_logger.warning('\t' + var_name)
laconic_logger.info('')
def print_layer_shapes(self):
laconic_logger.info('Net shapes:')
layers = get_all_layers(self._net['dist'])
for l in layers:
laconic_logger.info('\t%-20s \t%s' % (l.name, get_output_shape(l)))