def print_predictions(nn_model, contexts_token_ids, condition, prediction_mode=PREDICTION_MODE_FOR_TESTS): x_sents = transform_context_token_ids_to_sentences(contexts_token_ids, nn_model.index_to_token) y_sents = predict_for_condition_id( nn_model, contexts_token_ids, nn_model.condition_to_index[condition], prediction_mode=prediction_mode) for x, y in zip(x_sents, y_sents): print('condition: %s; context: %s' % (condition.encode('utf8'), x.encode('utf8'))) print('response: %s' % y.encode('utf8')) print()
def print_predictions(nn_model, contexts_token_ids, condition, prediction_mode=PREDICTION_MODE_FOR_TESTS): x_sents = transform_context_token_ids_to_sentences(contexts_token_ids, nn_model.index_to_token) y_sents = predict_for_condition_id( nn_model, contexts_token_ids, nn_model.condition_to_index[condition], prediction_mode=prediction_mode) for x, y in zip(x_sents, y_sents): print 'condition: %s; context: %s' % (condition.encode('utf8'), x.encode('utf8')) print 'response: %s' % y.encode('utf8') print
def _log_sample_answers(x_test, nn_model, mode, is_reverse_model): _logger.info('Model: {}'.format(get_model_full_path(is_reverse_model))) _logger.info('Start predicting responses of length {out_len} for {n_samples} samples with mode {mode}'.format( out_len=MAX_PREDICTIONS_LENGTH, n_samples=x_test.shape[0], mode=mode)) questions = transform_context_token_ids_to_sentences(x_test, nn_model.index_to_token) responses = get_nn_responses(x_test, nn_model, mode, output_candidates_num=LOG_CANDIDATES_NUM) _logger.info('Finished predicting! Logging...') for i, (question_ids, question) in enumerate(zip(x_test, questions)): laconic_logger.info('') # for better readability for j, response in enumerate(responses[i]): laconic_logger.info('%-35s\t --#=%02d--> \t%s' % (question, j + 1, response))
def log_predictions(predictions_path, x_test, nn_model, mode, candidates_num=None, stats_info=None, cur_perplexity=None, output_seq_len=MAX_PREDICTIONS_LENGTH, **kwargs): _logger.info('Logging responses to test lines') # Create all the directories for the prediction path in case they don't exist prediction_dir = os.path.dirname(predictions_path) if prediction_dir: ensure_dir(prediction_dir) with open(predictions_path, 'w') as test_res_fh: csv_writer = init_csv_writer(test_res_fh, mode, output_seq_len) if cur_perplexity: csv_writer.writerow(['Current perplexity: %.2f' % cur_perplexity]) if stats_info: csv_writer.writerow([_get_iteration_stats(stats_info)]) csv_writer.writerow( ['input sentence'] + ['candidate #{}'.format(v + 1) for v in xrange(candidates_num)]) questions = transform_context_token_ids_to_sentences( x_test, nn_model.index_to_token) _logger.info( 'Start predicting responses of length {out_len} for {n_samples} samples with mode {mode}' .format(out_len=output_seq_len, n_samples=x_test.shape[0], mode=mode)) nn_responses = get_nn_responses(x_test, nn_model, mode, candidates_num, output_seq_len, **kwargs) _logger.info('Logging generated predictions...') for idx, (question, responses) in enumerate(zip(questions, nn_responses)): csv_writer.writerow([question] + responses) _logger.info( 'Succesfully dumped {n_resp} of {n_resp} responses'.format( n_resp=len(questions))) _logger.info('Here they are: {}'.format(predictions_path))
def print_predictions(nn_model, contexts_token_ids, condition, prediction_mode=PREDICTION_MODE_FOR_TESTS): x_sents = transform_context_token_ids_to_sentences(contexts_token_ids, nn_model.index_to_token) y_sents = predict_for_condition_id(nn_model, contexts_token_ids, nn_model.condition_to_index[condition], prediction_mode=prediction_mode) for x, y in zip(x_sents, y_sents): print('condition: {}; context: {}'.format(condition, x)) print('response: {}'.format(y)) print()
def log_predictions(predictions_path, contexts_token_ids, nn_model, prediction_modes, output_seq_len=MAX_PREDICTIONS_LENGTH, **kwargs): """ Generate responses for provided contexts and save the results on the disk. For a given context several responses will be generated - one for each mode from the prediction_modes list. :param predictions_path: Generated responses will be saved to this file :param contexts_token_ids: contexts token ids, numpy array of shape (batch_size, context_len, INPUT_SEQUENCE_LENGTH) :param nn_model: instance of CakeChatModel class :param prediction_modes: See PREDICTION_MODES for available options :param output_seq_len: Max number of tokens in generated responses """ _logger.info('Logging responses for test set') # Create all the directories for the prediction path in case they don't exist ensure_dir(os.path.dirname(predictions_path)) _init_csv_writer(predictions_path, output_seq_len, nn_model.model_name) contexts = transform_context_token_ids_to_sentences( contexts_token_ids, nn_model.index_to_token) predictions_data = pd.DataFrame() predictions_data['contexts'] = contexts for prediction_mode in prediction_modes: predicted_responses = get_nn_responses(contexts_token_ids, nn_model, prediction_mode, **kwargs) # list of lists of strings, shape (contexts_num, 1) predicted_responses = [response[0] for response in predicted_responses] # list of strings, shape (contexts_num) predictions_data[prediction_mode] = predicted_responses predictions_data.to_csv(predictions_path, sep='\t', index=False, encoding='utf-8', mode='a', float_format='%.2f') _logger.info('Dumped {} predicted responses to {}'.format( len(contexts), predictions_path))
def log_predictions(predictions_path, x_test, nn_model, mode, candidates_num=None, stats_info=None, cur_perplexity=None, output_seq_len=MAX_PREDICTIONS_LENGTH, **kwargs): _logger.info('Logging responses to test lines') # Create all the directories for the prediction path in case they don't exist prediction_dir = os.path.dirname(predictions_path) if prediction_dir: ensure_dir(prediction_dir) with open(predictions_path, 'w') as test_res_fh: csv_writer = init_csv_writer(test_res_fh, mode, output_seq_len) if cur_perplexity: csv_writer.writerow(['Current perplexity: %.2f' % cur_perplexity]) if stats_info: csv_writer.writerow([_get_iteration_stats(stats_info)]) csv_writer.writerow(['input sentence'] + ['candidate #{}'.format(v + 1) for v in xrange(candidates_num)]) questions = transform_context_token_ids_to_sentences(x_test, nn_model.index_to_token) _logger.info('Start predicting responses of length {out_len} for {n_samples} samples with mode {mode}'.format( out_len=output_seq_len, n_samples=x_test.shape[0], mode=mode)) nn_responses = get_nn_responses(x_test, nn_model, mode, candidates_num, output_seq_len, **kwargs) _logger.info('Logging generated predictions...') for idx, (question, responses) in enumerate(zip(questions, nn_responses)): csv_writer.writerow([question] + responses) _logger.info('Succesfully dumped {n_resp} of {n_resp} responses'.format(n_resp=len(questions))) _logger.info('Here they are: {}'.format(predictions_path))
def __init__(self, model, index_to_token, batch_size, batches_num_per_epoch, eval_state_per_batches=EVAL_STATE_PER_BATCHES, prediction_mode_for_tests=PREDICTION_MODE_FOR_TESTS, log_run_metadata=LOG_RUN_METADATA, screen_log_num_test_lines=SCREEN_LOG_NUM_TEST_LINES, log_candidates_num=LOG_CANDIDATES_NUM): """ :param model: CakeChatModel object :param eval_state_per_batches: run model evaluation each `eval_state_per_batches` steps """ super(CakeChatEvaluatorCallback, self).__init__(model, eval_state_per_batches) self._index_to_token = index_to_token self._token_to_index = {v: k for k, v in index_to_token.items()} self._val_contexts_tokens_ids = load_questions_set(self._token_to_index).x[:screen_log_num_test_lines] self._val_contexts = \ transform_context_token_ids_to_sentences(self._val_contexts_tokens_ids, self._index_to_token) self._batch_size = batch_size self._batches_num = batches_num_per_epoch self._cur_batch_id = 0 self._cur_loss = 0 self._batch_start_time = None self._total_training_time = 0 # logging params self._prediction_mode_for_tests = prediction_mode_for_tests self._log_run_metadata = log_run_metadata self._log_candidates_num = log_candidates_num
def log_predictions(predictions_path, x_test, nn_model, prediction_modes=(PREDICTION_MODE_FOR_TESTS, ), stats_info=None, cur_perplexity=None, output_seq_len=MAX_PREDICTIONS_LENGTH, **kwargs): """ Generate responses for provided contexts and save the results on the disk. For a given context several responses will be generated - one for each mode from the prediction_modes list. :param predictions_path: Generated responses will be saved to this file :param x_test: context token ids, numpy array of shape (batch_size, context_len, INPUT_SEQUENCE_LENGTH) :param nn_model: instance of CakeChatModel class :param prediction_modes: Iterable of modes to be used for responses generation. See PREDICTION_MODES for available options :param stats_info: Info about current training status: total time passed, time spent on training, processed batches number and estimated time for one epoch :param cur_perplexity: Addition to stats_info - current perplexity metric calculated on validation dataset :param output_seq_len: Max number of tokens in generated responses """ _logger.info('Logging responses to test lines') # Create all the directories for the prediction path in case they don't exist prediction_dir = os.path.dirname(predictions_path) if prediction_dir: ensure_dir(prediction_dir) with open(predictions_path, 'w') as test_res_fh: csv_writer = init_csv_writer(test_res_fh, output_seq_len, nn_model.model_name) if cur_perplexity: csv_writer.writerow(['Current perplexity: %.2f' % cur_perplexity]) if stats_info: csv_writer.writerow([_get_iteration_stats(stats_info)]) contexts = transform_context_token_ids_to_sentences( x_test, nn_model.index_to_token) predictions_data = pd.DataFrame() predictions_data['contexts'] = contexts for pred_mode in prediction_modes: responses_batch = get_nn_responses(x_test, nn_model, pred_mode, **kwargs) # list of lists of strings, shape (contexts_num, 1) first_responses_batch = [response[0] for response in responses_batch] # list of strings, shape (contexts_num) predictions_data[pred_mode] = first_responses_batch predictions_data.to_csv(predictions_path, sep='\t', index=False, encoding='utf-8', mode='a', float_format='%.2f') message = '\nSuccesfully dumped {} responses.'.format(len(contexts)) message += '\nHere they are:\n{}\n'.format(predictions_path) _logger.info(message)