def _load_dataset_without_responses(corpus_name, token_to_index): tokenized_lines = get_tokenized_test_lines(corpus_name, set(token_to_index.keys())) context_tokens_ids = transform_contexts_to_token_ids( lines_to_context(tokenized_lines), token_to_index, INPUT_SEQUENCE_LENGTH, INPUT_CONTEXT_SIZE, max_contexts_num=len(tokenized_lines)) return Dataset(x=context_tokens_ids, y=None, condition_ids=None)
def _save_test_results(test_dataset, predictions_filename, nn_model, prediction_modes, **kwargs): test_dataset_ids = transform_contexts_to_token_ids( list(lines_to_context(test_dataset)), nn_model.token_to_index, INPUT_SEQUENCE_LENGTH, INPUT_CONTEXT_SIZE) calculate_and_log_val_metrics( nn_model, load_context_sensitive_val(nn_model.token_to_index, nn_model.condition_to_index), load_context_free_val(nn_model.token_to_index)) log_predictions(predictions_filename, test_dataset_ids, nn_model, prediction_modes, **kwargs)
def _save_test_results(test_dataset, predictions_filename, nn_model, prediction_mode, **kwargs): test_dataset_ids = transform_contexts_to_token_ids( list(lines_to_context(test_dataset)), nn_model.token_to_index, INPUT_SEQUENCE_LENGTH, INPUT_CONTEXT_SIZE) calculate_and_log_val_metrics(nn_model, load_context_sensitive_val(nn_model.token_to_index, nn_model.condition_to_index), load_context_free_val(nn_model.token_to_index)) log_predictions( predictions_filename, test_dataset_ids, nn_model, mode=prediction_mode, candidates_num=LOG_CANDIDATES_NUM, **kwargs)
def _get_context_to_weighted_responses(nn_model, testset, all_utterances): token_to_index = nn_model.token_to_index all_utterances_ids = transform_lines_to_token_ids( map(get_tokens_sequence, all_utterances), token_to_index, OUTPUT_SEQUENCE_LENGTH, add_start_end=True) context_to_weighted_responses = {} for context in testset: context_tokenized = get_tokens_sequence(context) repeated_context_ids = transform_contexts_to_token_ids( [[context_tokenized]] * len(all_utterances), token_to_index, INPUT_SEQUENCE_LENGTH, INPUT_CONTEXT_SIZE) scores = get_sequence_score(nn_model, repeated_context_ids, all_utterances_ids) context_to_weighted_responses[context] = dict(zip(all_utterances, scores)) return context_to_weighted_responses
def get_response(dialog_context, emotion): """ :param dialog_context: list of dialog utterances :param emotion: emotion to condition response :return: dialog response conditioned on input emotion """ tokenized_dialog_context = list(map(get_tokens_sequence, dialog_context)) tokenized_dialog_contexts = [tokenized_dialog_context] context_tokens_ids = transform_contexts_to_token_ids( tokenized_dialog_contexts, _cakechat_model.token_to_index, INPUT_SEQUENCE_LENGTH, INPUT_CONTEXT_SIZE) condition_ids = transform_conditions_to_ids( [emotion], _cakechat_model.condition_to_index, n_dialogs=1) if PREDICTION_MODE == PREDICTION_MODES.sampling: # Different strategy here for better performance. return _get_non_offensive_response_using_fast_sampling( context_tokens_ids, condition_ids) else: return _get_non_offensive_response(context_tokens_ids, condition_ids)
def _get_context_to_weighted_responses(nn_model, testset, all_utterances): token_to_index = nn_model.token_to_index all_utterances_ids = transform_lines_to_token_ids(list( map(get_tokens_sequence, all_utterances)), token_to_index, OUTPUT_SEQUENCE_LENGTH, add_start_end=True) context_to_weighted_responses = {} for context in testset: context_tokenized = get_tokens_sequence(context) repeated_context_ids = transform_contexts_to_token_ids( [[context_tokenized]] * len(all_utterances), token_to_index, INPUT_SEQUENCE_LENGTH, INPUT_CONTEXT_SIZE) scores = get_sequence_score(nn_model, repeated_context_ids, all_utterances_ids) context_to_weighted_responses[context] = dict( zip(all_utterances, scores)) return context_to_weighted_responses
def transform_lines_to_contexts_token_ids(tokenized_lines, nn_model): return transform_contexts_to_token_ids( list(lines_to_context(tokenized_lines)), nn_model.token_to_index, INPUT_SEQUENCE_LENGTH, INPUT_CONTEXT_SIZE)
def transform_lines_to_contexts_token_ids(tokenized_lines, nn_model): return transform_contexts_to_token_ids( list(lines_to_context(tokenized_lines)), nn_model.token_to_index, INPUT_SEQUENCE_LENGTH, INPUT_CONTEXT_SIZE)