def test_one_step_decoder(self): nn_model = get_trained_model() _EPS = 1e-6 batch_size = 1 context_size = 3 input_seq_len = 10 output_seq_len = 9 x = np.random.randint(0, nn_model.vocab_size, size=(batch_size, context_size, input_seq_len), dtype=np.int32) y = np.random.randint(0, nn_model.vocab_size, size=(batch_size, output_seq_len), dtype=np.int32) ground_truth_log_probabilities = get_sequence_log_probs( nn_model, x, y, condition_ids=None) one_step_log_probabilities = self._predict_log_probabilities_one_step( nn_model, x, y) mae = np.abs(one_step_log_probabilities - ground_truth_log_probabilities).mean() self.assertTrue(mae < _EPS)
def test_one_step_decoder(self): nn_model = get_trained_model() _EPS = 1e-5 batch_size = 1 # input batches shapes should correspond to the shapes of the trained model layers context_size = INPUT_CONTEXT_SIZE input_seq_len = INPUT_SEQUENCE_LENGTH output_seq_len = OUTPUT_SEQUENCE_LENGTH x = np.random.randint(0, nn_model.vocab_size, size=(batch_size, context_size, input_seq_len), dtype=INTX) y = np.random.randint(0, nn_model.vocab_size, size=(batch_size, output_seq_len), dtype=INTX) ground_truth_log_probabilities = get_sequence_log_probs( nn_model, x, y, condition_ids=None) one_step_log_probabilities = self._predict_log_probabilities_one_step( nn_model, x, y) mae = np.abs(one_step_log_probabilities - ground_truth_log_probabilities).mean() self.assertTrue(mae < _EPS)
def _get_reverse_model(): if not hasattr(_get_reverse_model, 'reverse_model'): try: reverse_model = get_trained_model(reverse=True) except: raise ValueError('Can\'t get reverse nn model for prediction. ' 'Try to run \'python tools/train.py --reverse\' or switch prediction mode to sampling.') _get_reverse_model.reverse_model = reverse_model return _get_reverse_model.reverse_model
def evaluate_distinctness(args): if args.sample_size > 1 and PREDICTION_MODE_FOR_TESTS == PREDICTION_MODES.beamsearch: _logger.waring( 'Using sample_size > 1 is meaningless with prediction_mode=\'beamsearch\' because there\'s no ' 'randomness in the prediction. Use sample_size=1 instead.') nn_model = get_trained_model() if args.validation_only: validation = load_questions_set(nn_model.token_to_index) validation_set_name = 'context free questions' else: eval_datasets = load_datasets(nn_model.token_to_index, nn_model.condition_to_index) validation = eval_datasets.cf_validation cs_test = eval_datasets.cs_test cs_test_one_condition = eval_datasets.cs_test_one_condition validation_set_name = 'validation set without conditions' _logger.info( 'Evaluating distinctness for context sensitive testset without conditions' ) log_distinct_metrics(nn_model, cs_test.x, samples_num=args.sample_size) _logger.info( 'Evaluating distinctness for context sensitive testset with conditions' ) log_distinct_metrics(nn_model, cs_test.x, cs_test.condition_ids, samples_num=args.sample_size) _logger.info( 'Evaluating distinctness for defined-conditions-subset without conditions' ) log_distinct_metrics(nn_model, cs_test_one_condition.x, samples_num=args.sample_size) _logger.info( 'Evaluating distinctness for defined-conditions-subset with conditions' ) log_distinct_metrics(nn_model, cs_test_one_condition.x, cs_test_one_condition.condition_ids, samples_num=args.sample_size) _logger.info('Evaluating distinctness for {}'.format(validation_set_name)) log_distinct_metrics(nn_model, validation.x, samples_num=args.sample_size)
def test_one_step_decoder(self): nn_model = get_trained_model() _EPS = 1e-6 batch_size = 1 context_size = 3 input_seq_len = 10 output_seq_len = 9 x = np.random.randint(0, nn_model.vocab_size, size=(batch_size, context_size, input_seq_len), dtype=np.int32) y = np.random.randint(0, nn_model.vocab_size, size=(batch_size, output_seq_len), dtype=np.int32) ground_truth_log_probabilities = get_sequence_log_probs(nn_model, x, y, condition_ids=None) one_step_log_probabilities = self._predict_log_probabilities_one_step(nn_model, x, y) mae = np.abs(one_step_log_probabilities - ground_truth_log_probabilities).mean() self.assertTrue(mae < _EPS)
def predictor_factory(nn_model, mode, config): """ :param nn_model: Model used for predicting :param mode: Prediction mode: 'sampling', 'sampling-reranking' or 'candidates' :param config: All additional prediction parameters. See PredictionConfig for the details. :return: BasePredictor descendant with predict_response() method implemented. """ if mode not in PREDICTION_MODES: raise ValueError( 'Unknown prediction mode {}. Use one of the following: {}.'.format( mode, list(PREDICTION_MODES))) if mode in [ PREDICTION_MODES.beamsearch, PREDICTION_MODES.beamsearch_reranking ]: candidates_generator = BeamsearchCandidatesGenerator( nn_model, config['beam_size'], config['repetition_penalization_coefficient']) else: candidates_generator = SamplingCandidatesGenerator( nn_model, config['temperature'], config['samples_num'], config['repetition_penalization_coefficient']) if mode in [ PREDICTION_MODES.beamsearch_reranking, PREDICTION_MODES.sampling_reranking ]: if config['mmi_reverse_model_score_weight'] <= 0: raise ValueError( 'mmi_reverse_model_score_weight should be > 0 for reranking mode' ) reverse_model = get_trained_model(reverse=True) reranker = MMIReranker(nn_model, reverse_model, config['mmi_reverse_model_score_weight'], config['repetition_penalization_coefficient']) else: reranker = DummyReranker() return Predictor(nn_model, candidates_generator, reranker)
argparser.add_argument('-t', '--text', action='store', help='Context message that feed to the model', default=None) argparser.add_argument('-c', '--condition', action='store', help='Condition', default=DEFAULT_CONDITION) return argparser.parse_args() if __name__ == '__main__': args = parse_args() nn_model = get_trained_model() if args.text: tokenized_lines = process_text(nn_model, args.text.decode('utf8')) else: tokenized_lines = load_corpus(nn_model, args.data) contexts_token_ids = transform_lines_to_contexts_token_ids( tokenized_lines, nn_model) print_predictions(nn_model, contexts_token_ids, args.condition, prediction_mode=args.prediction_mode)
from cakechat.utils.w2v.model import get_w2v_model _logger = get_tools_logger(__file__) def parse_args(): argparser = argparse.ArgumentParser() argparser.add_argument( '-m', '--model', action='store', choices=['default', 'reverse', 'w2v', 'all'], help='Fetch models from s3 to disk', default='all') args = argparser.parse_args() return args if __name__ == '__main__': args = parse_args() if args.model in {'default', 'all'}: get_trained_model(fetch_from_s3=True) if args.model in {'reverse', 'all'}: get_trained_model(fetch_from_s3=True, is_reverse_model=True) if args.model in {'w2v', 'all'}: get_w2v_model(fetch_from_s3=True)
import random from cakechat.api.config import PREDICTION_MODE, NUM_BEST_CANDIDATES_TO_PICK_FROM, SAMPLING_ATTEMPTS_NUM, \ DEFAULT_RESPONSE from cakechat.config import INPUT_CONTEXT_SIZE, INPUT_SEQUENCE_LENGTH, PREDICTION_MODES from cakechat.dialog_model.factory import get_trained_model from cakechat.dialog_model.inference import get_nn_responses, warmup_predictor from cakechat.dialog_model.model_utils import transform_contexts_to_token_ids, transform_conditions_to_ids from cakechat.utils.offense_detector.config import OFFENSIVE_PHRASES_PATH from cakechat.utils.offense_detector import OffenseDetector from cakechat.utils.text_processing import get_tokens_sequence, get_pretty_str_from_tokens_sequence _offense_detector = OffenseDetector(OFFENSIVE_PHRASES_PATH) _cakechat_model = get_trained_model(fetch_from_s3=False) warmup_predictor(_cakechat_model, PREDICTION_MODE) def _get_non_offensive_response_using_fast_sampling(context_tokens_ids, condition_id): for _ in xrange(SAMPLING_ATTEMPTS_NUM): response = get_nn_responses(context_tokens_ids, _cakechat_model, PREDICTION_MODES.sampling, condition_ids=condition_id)[0][0] tokenized_response = get_tokens_sequence(response) if not _offense_detector.has_offensive_ngrams(tokenized_response): return get_pretty_str_from_tokens_sequence(tokenized_response) return DEFAULT_RESPONSE
continue responses_ground_truth = transform_token_ids_to_sentences(responses_token_ids_ground_truth, nn_model.index_to_token) responses = predict_for_condition_id(nn_model, questions.x, condition_id) lex_sim_conditioned_vs_non_conditioned = calculate_lexical_similarity(responses, responses_baseline, tfidf_vectorizer) lex_sim_conditioned_vs_groundtruth = calculate_lexical_similarity(responses, responses_ground_truth, tfidf_vectorizer) yield condition, (lex_sim_conditioned_vs_non_conditioned, lex_sim_conditioned_vs_groundtruth) if __name__ == '__main__': nn_model = get_trained_model() train, questions, validation, train_subset, conditioned_subset = load_datasets(nn_model.token_to_index, nn_model.condition_to_index) tfidf_vectorizer = get_tfidf_vectorizer() for metric, perplexity in calc_perplexity_metrics(nn_model, train_subset, conditioned_subset, validation).iteritems(): _logger.info('Metric: {}, perplexity: {}'.format(metric, perplexity)) for condition, (ppl_non_conditioned, ppl_conditioned) in calc_perplexity_by_condition_metrics(nn_model, train): _logger.info('Condition: {}, non-conditioned perplexity: {}, conditioned perplexity: {}'.format( condition, ppl_non_conditioned, ppl_conditioned)) for condition, (lex_sim_conditioned_vs_non_conditioned, lex_sim_conditioned_vs_groundtruth) in \ calc_lexical_similarity_metrics(nn_model, train, questions, tfidf_vectorizer): _logger.info('Condition: {}, conditioned vs non-conditioned lexical similarity: {}'.format(
#!/usr/bin/env python """ Gets trained model and warms it up (i.e. compiles and dumps corresponding prediction functions) """ import os import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from cakechat.utils.env import init_theano_env init_theano_env() from cakechat.dialog_model.factory import get_trained_model from cakechat.utils.logger import get_tools_logger _logger = get_tools_logger(__file__) if __name__ == '__main__': _logger.info('Fetching and pre-compiling pre-trained model...') get_trained_model(fetch_from_s3=True) _logger.info('Successfully resolved and compiled model.') _logger.info('Fetching and pre-compiling additional reverse-model for MMI reranking...') get_trained_model(fetch_from_s3=True, reverse=True) _logger.info('Successfully resolved and compiled reverse-model.')
import random from cakechat.api.config import PREDICTION_MODE, NUM_BEST_CANDIDATES_TO_PICK_FROM, SAMPLING_ATTEMPTS_NUM, \ DEFAULT_RESPONSE from cakechat.config import INPUT_CONTEXT_SIZE, INPUT_SEQUENCE_LENGTH, PREDICTION_MODES from cakechat.dialog_model.factory import get_trained_model, get_reverse_model from cakechat.dialog_model.inference import get_nn_responses, warmup_predictor from cakechat.dialog_model.model_utils import transform_contexts_to_token_ids, transform_conditions_to_ids from cakechat.utils.offense_detector import OffenseDetector from cakechat.utils.offense_detector.config import OFFENSIVE_PHRASES_PATH from cakechat.utils.text_processing import get_tokens_sequence, prettify_response _offense_detector = OffenseDetector(OFFENSIVE_PHRASES_PATH) _cakechat_model = get_trained_model( reverse_model=get_reverse_model(PREDICTION_MODE)) warmup_predictor(_cakechat_model, PREDICTION_MODE) def _is_appropriate_response(response): return response != '' and not _offense_detector.has_offensive_ngrams( response) def _get_non_offensive_response_using_fast_sampling(context_tokens_ids, condition_id): for _ in range(SAMPLING_ATTEMPTS_NUM): response = get_nn_responses(context_tokens_ids, _cakechat_model, PREDICTION_MODES.sampling, condition_ids=condition_id)[0][0]