def translate(src_seq, tokenized_input, params_prediction, index2word_y): # 1. Get a first hypothesis trans_indices, costs, alphas = interactive_beam_searcher.sample_beam_search_interactive( src_seq) # 1.1 Set unk replacemet strategy if params_prediction['pos_unk']: alphas = [alphas] sources = [tokenized_input] heuristic = params_prediction['heuristic'] else: alphas = None heuristic = None sources = None # 1.2 Decode hypothesis hypothesis = decode_predictions_beam_search([trans_indices], index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=mapping, pad_sequences=True, verbose=0)[0] hypothesis = params_prediction['detokenize_f']( hypothesis) if params_prediction.get('apply_detokenization', False) else hypothesis return hypothesis, costs, alphas, trans_indices
def check_grammar(self, input_sentence): with self.graph.as_default(): with self.session.as_default(): user_input = input_sentence with open('user_input.txt', 'w') as f: f.write(user_input) self.dataset.setInput('user_input.txt', 'test', type='text', id='source_text', pad_on_batch=True, tokenization='tokenize_basic', fill='end', max_text_len=30, min_occ=0, overwrite_split=True) self.dataset.setInput(None, 'test', type='ghost', id='state_below', required=False, overwrite_split=True) self.dataset.setRawInput('user_input.txt', 'test', type='file-name', id='raw_source_text', overwrite_split=True) vocab = self.dataset.vocabulary['target_text']['idx2words'] predictions = self.nmt_model.predictBeamSearchNet( self.dataset, self.params_prediction)['test'] if self.params_prediction['pos_unk']: samples = predictions['samples'] alphas = predictions['alphas'] else: samples = predictions heuristic = None sources = None predictions = decode_predictions_beam_search( samples, # The first element of predictions contain the word indices. vocab, verbose=self.params['VERBOSE']) #print("Correct Prediction: {}".format(predictions[0])) return predictions[0]
def predictResponse(self, context): with self.graph.as_default(): with self.session.as_default(): with open(os.path.join(MODEL_PATH, 'context.txt'), 'w') as f: f.write(context) self.dataset.setInput(os.path.join(MODEL_PATH, 'context.txt'), 'test', type='text', id='source_text', pad_on_batch=True, tokenization='tokenize_basic', fill='end', max_text_len=30, min_occ=0, overwrite_split=True) self.dataset.setInput(None, 'test', type='ghost', id='state_below', required=False, overwrite_split=True) self.dataset.setRawInput(os.path.join(MODEL_PATH, 'context.txt'), 'test', type='file-name', id='raw_source_text', overwrite_split=True) vocab = self.dataset.vocabulary['target_text']['idx2words'] predictions = self.model.predictBeamSearchNet(self.dataset, self.params)['test'] if self.params['pos_unk']: samples = predictions['samples'] alphas = predictions['alphas'] else: samples = predictions heuristic = None sources = None predictions = decode_predictions_beam_search(samples, vocab) print('prediction: ' + predictions[0]) return predictions[0]
id='raw_source_text', overwrite_split=True) vocab = dataset.vocabulary['target_text']['idx2words'] predictions = nmt_model.predictBeamSearchNet(dataset, params_prediction)['test'] if params_prediction['pos_unk']: samples = predictions['samples'] # The first element of predictions contain the word indices. alphas = predictions['alphas'] else: samples = predictions heuristic = None sources = None predictions = decode_predictions_beam_search(samples, vocab, verbose=params['VERBOSE']) print(predictions[0]) bot_responses.append(predictions[0]) if (len(context) > 2): context.pop(0) context.append(predictions[0]) # text = open(os.path.join(DATA_PATH, 'train_y.txt')).read() # lines = text.split('\n') # for i, line in enumerate(lines): # print('y_true: ' + line + '\t\ty_pred: ' + predictions[i]) # filepath = '/content/drive/My Drive/test/user_input_preds.txt' # list2file(filepath, predictions) # with open(filepath, 'r') as f: # pred = f.readline() # print(pred)
def sample_ensemble(args, params): from data_engine.prepare_data import update_dataset_from_file from keras_wrapper.model_ensemble import BeamSearchEnsemble from keras_wrapper.cnn_model import loadModel from keras_wrapper.dataset import loadDataset from keras_wrapper.utils import decode_predictions_beam_search logging.info("Using an ensemble of %d models" % len(args.models)) models = [loadModel(m, -1, full_path=True) for m in args.models] dataset = loadDataset(args.dataset) dataset = update_dataset_from_file(dataset, args.text, params, splits=args.splits, remove_outputs=True) params['INPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len[ params['INPUTS_IDS_DATASET'][0]] params['OUTPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len[ params['OUTPUTS_IDS_DATASET'][0]] # For converting predictions into sentences index2word_y = dataset.vocabulary[params['OUTPUTS_IDS_DATASET'] [0]]['idx2words'] if params.get('APPLY_DETOKENIZATION', False): detokenize_function = eval('dataset.' + params['DETOKENIZATION_METHOD']) params_prediction = dict() params_prediction['max_batch_size'] = params.get('BATCH_SIZE', 20) params_prediction['n_parallel_loaders'] = params.get('PARALLEL_LOADERS', 1) params_prediction['beam_size'] = params.get('BEAM_SIZE', 6) params_prediction['maxlen'] = params.get('MAX_OUTPUT_TEXT_LEN_TEST', 100) params_prediction['optimized_search'] = params['OPTIMIZED_SEARCH'] params_prediction['model_inputs'] = params['INPUTS_IDS_MODEL'] params_prediction['model_outputs'] = params['OUTPUTS_IDS_MODEL'] params_prediction['dataset_inputs'] = params['INPUTS_IDS_DATASET'] params_prediction['dataset_outputs'] = params['OUTPUTS_IDS_DATASET'] params_prediction['search_pruning'] = params.get('SEARCH_PRUNING', False) params_prediction['normalize_probs'] = params.get('NORMALIZE_SAMPLING', False) params_prediction['alpha_factor'] = params.get('ALPHA_FACTOR', 1.0) params_prediction['coverage_penalty'] = params.get('COVERAGE_PENALTY', False) params_prediction['length_penalty'] = params.get('LENGTH_PENALTY', False) params_prediction['length_norm_factor'] = params.get( 'LENGTH_NORM_FACTOR', 0.0) params_prediction['coverage_norm_factor'] = params.get( 'COVERAGE_NORM_FACTOR', 0.0) params_prediction['pos_unk'] = params.get('POS_UNK', False) params_prediction['state_below_maxlen'] = -1 if params.get('PAD_ON_BATCH', True) \ else params.get('MAX_OUTPUT_TEXT_LEN', 50) params_prediction['output_max_length_depending_on_x'] = params.get( 'MAXLEN_GIVEN_X', True) params_prediction['output_max_length_depending_on_x_factor'] = params.get( 'MAXLEN_GIVEN_X_FACTOR', 3) params_prediction['output_min_length_depending_on_x'] = params.get( 'MINLEN_GIVEN_X', True) params_prediction['output_min_length_depending_on_x_factor'] = params.get( 'MINLEN_GIVEN_X_FACTOR', 2) params_prediction['attend_on_output'] = params.get( 'ATTEND_ON_OUTPUT', 'transformer' in params['MODEL_TYPE'].lower()) heuristic = params.get('HEURISTIC', 0) mapping = None if dataset.mapping == dict() else dataset.mapping model_weights = args.weights if model_weights is not None and model_weights != []: assert len(model_weights) == len( models ), 'You should give a weight to each model. You gave %d models and %d weights.' % ( len(models), len(model_weights)) model_weights = map(lambda x: float(x), model_weights) if len(model_weights) > 1: logger.info('Giving the following weights to each model: %s' % str(model_weights)) for s in args.splits: # Apply model predictions params_prediction['predict_on_sets'] = [s] beam_searcher = BeamSearchEnsemble(models, dataset, params_prediction, model_weights=model_weights, n_best=args.n_best, verbose=args.verbose) if args.n_best: predictions, n_best = beam_searcher.predictBeamSearchNet()[s] else: predictions = beam_searcher.predictBeamSearchNet()[s] n_best = None if params_prediction['pos_unk']: samples = predictions[0] alphas = predictions[1] sources = [ x.strip() for x in open(args.text, 'r').read().split('\n') ] sources = sources[:-1] if len(sources[-1]) == 0 else sources else: samples = predictions alphas = None heuristic = None sources = None predictions = decode_predictions_beam_search(samples, index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=mapping, verbose=args.verbose) # Apply detokenization function if needed if params.get('APPLY_DETOKENIZATION', False): predictions = map(detokenize_function, predictions) if args.n_best: n_best_predictions = [] for i, (n_best_preds, n_best_scores, n_best_alphas) in enumerate(n_best): n_best_sample_score = [] for n_best_pred, n_best_score, n_best_alpha in zip( n_best_preds, n_best_scores, n_best_alphas): pred = decode_predictions_beam_search( [n_best_pred], index2word_y, alphas=[n_best_alpha] if params_prediction['pos_unk'] else None, x_text=[sources[i]] if params_prediction['pos_unk'] else None, heuristic=heuristic, mapping=mapping, verbose=args.verbose) # Apply detokenization function if needed if params.get('APPLY_DETOKENIZATION', False): pred = map(detokenize_function, pred) n_best_sample_score.append([i, pred, n_best_score]) n_best_predictions.append(n_best_sample_score) # Store result if args.dest is not None: filepath = args.dest # results file if params.get('SAMPLING_SAVE_MODE', 'list'): list2file(filepath, predictions) if args.n_best: nbest2file(filepath + '.nbest', n_best_predictions) else: raise Exception( 'Only "list" is allowed in "SAMPLING_SAVE_MODE"') else: list2stdout(predictions) if args.n_best: logging.info('Storing n-best sentences in ./' + s + '.nbest') nbest2file('./' + s + '.nbest', n_best_predictions) logging.info('Sampling finished')
def on_batch_end(self, n_update, logs=None): self.cum_update += 1 if self.epoch_count + self.reload_epoch < self.start_sampling_on_epoch: return elif self.cum_update % self.each_n_updates != 0: return # Evaluate on each set separately for s in self.set_name: if self.beam_search: params_prediction = {'max_batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars[ 'n_parallel_loaders'], 'predict_on_sets': [s], 'n_samples': self.n_samples, 'pos_unk': False} params_prediction.update(checkDefaultParamsBeamSearch(self.extra_vars)) predictions, truths, sources = self.model_to_eval.predictBeamSearchNet(self.ds, params_prediction) else: params_prediction = {'batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars[ 'n_parallel_loaders'], 'predict_on_sets': [s], 'n_samples': self.n_samples, } # Convert predictions postprocess_fun = None if self.is_3DLabel: postprocess_fun = [self.ds.convert_3DLabels_to_bboxes, self.extra_vars[s]['references_orig_sizes']] predictions = self.model_to_eval.predictNet(self.ds, params_prediction, postprocess_fun=postprocess_fun) if self.print_sources: if self.in_pred_idx is not None: sources = [srcs[self.in_pred_idx][0] for srcs in sources] sources = decode_predictions_beam_search(sources, self.index2word_x, pad_sequences=True, verbose=self.verbose) if s in predictions: if params_prediction['pos_unk']: samples = predictions[s][0] alphas = predictions[s][1] heuristic = self.extra_vars['heuristic'] else: samples = predictions[s] alphas = None heuristic = None predictions = predictions[s] if self.is_text: if self.out_pred_idx is not None: samples = samples[self.out_pred_idx] # Convert predictions into sentences if self.beam_search: predictions = decode_predictions_beam_search(samples, self.index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=self.extra_vars.get('mapping', None), verbose=self.verbose) else: predictions = decode_predictions(samples, 1, self.index2word_y, self.sampling_type, verbose=self.verbose) truths = decode_predictions_one_hot(truths, self.index2word_y, verbose=self.verbose) # Apply detokenization function if needed if self.extra_vars.get('apply_detokenization', False): if self.print_sources: sources = map(self.extra_vars['detokenize_f'], sources) predictions = map(self.extra_vars['detokenize_f'], predictions) truths = map(self.extra_vars['detokenize_f'], truths) # Write samples if self.print_sources: # Write samples for i, (source, sample, truth) in list(enumerate(zip(sources, predictions, truths))): if sys.version_info.major == 2: source = str(source.encode('utf-8')) sample = str(sample.encode('utf-8')) truth = str(truth.encode('utf-8')) print("Source (%d): %s" % (i, source)) print("Hypothesis (%d): %s" % (i, sample)) print("Reference (%d): %s" % (i, truth)) print("") else: for i, (sample, truth) in list(enumerate(zip(predictions, truths))): if sys.version_info.major == 2: sample = str(sample.encode('utf-8')) truth = str(truth.encode('utf-8')) print("Hypothesis (%d): %s" % (i, sample)) print("Reference (%d): %s" % (i, truth)) print("")
def generate_constrained_hypothesis(beam_searcher, src_seq, fixed_words_user, params, args, isle_indices, filtered_idx2word, index2word_y, sources, heuristic, mapping, unk_indices, unk_words, unks_in_isles, unk_id=1): """ Generates and decodes a user-constrained hypothesis given a source sentence and the user feedback signals. :param src_seq: Sequence of indices of the source sentence to translate. :param fixed_words_user: Dict of word indices fixed by the user and its corresponding position: {pos: idx_word} :param args: Simulation options :param isle_indices: Isles fixed by the user. List of (isle_index, [words]) :param filtered_idx2word: Dictionary of possible words according to the current word prefix. :param index2word_y: Indices to words mapping. :param sources: Source words (for unk replacement) :param heuristic: Unk replacement heuristic :param mapping: Source--Target dictionary for Unk replacement strategies 1 and 2 :param unk_indices: Indices of the hypothesis that contain an unknown word (introduced by the user) :param unk_words: Corresponding word for unk_indices :return: Constrained hypothesis """ # Generate a constrained hypothesis trans_indices, costs, alphas = beam_searcher. \ sample_beam_search_interactive(src_seq, fixed_words=copy.copy(fixed_words_user), max_N=args.max_n, isles=isle_indices, valid_next_words=filtered_idx2word, idx2word=index2word_y) # Substitute possible unknown words in isles unk_in_isles = [] for isle_idx, isle_sequence, isle_words in unks_in_isles: if unk_id in isle_sequence: unk_in_isles.append((subfinder(isle_sequence, list(trans_indices)), isle_words)) if params.get('pos_unk', False): alphas = [alphas] else: alphas = None # Decode predictions hypothesis = decode_predictions_beam_search([trans_indices], index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=mapping, pad_sequences=True, verbose=0)[0] hypothesis = hypothesis.split() for (words_idx, starting_pos), words in unk_in_isles: for pos_unk_word, pos_hypothesis in enumerate( range(starting_pos, starting_pos + len(words_idx))): hypothesis[pos_hypothesis] = words[pos_unk_word] # UNK words management if len(unk_indices) > 0: # If we added some UNK word if len(hypothesis) < len( unk_indices): # The full hypothesis will be made up UNK words: for i, index in enumerate(range(0, len(hypothesis))): hypothesis[index] = unk_words[unk_indices[i]] for ii in range(i + 1, len(unk_words)): hypothesis.append(unk_words[ii]) else: # We put each unknown word in the corresponding gap for i, index in list(enumerate(unk_indices)): if index < len(hypothesis): hypothesis[index] = list(unk_words)[i] else: hypothesis.append(unk_words[i]) return hypothesis
'beam_size': 12, 'maxlen': 50, 'model_inputs': ['source_text', 'state_below'], 'model_outputs': ['target_text'], 'dataset_inputs': ['source_text', 'state_below'], 'dataset_outputs': ['target_text'], 'normalize': True, 'alpha_factor': 0.6 } Control_predictions = Control_model.predictBeamSearchNet( dataset, params_prediction)['test'] vocab = dataset.vocabulary['target_text']['idx2words'] Control_predictions = decode_predictions_beam_search(Control_predictions, vocab, verbose=params['VERBOSE']) ## see how they compare to ground truth # from keras_wrapper.extra.read_write import list2file from keras_wrapper.extra import evaluation Control_path = 'Control_M7.pred' list2file(Control_path, Control_predictions) dataset.setOutput('data/Ross_test.reply', 'test', type='text', id='target_text', pad_on_batch=True,
def evaluate(self, epoch, counter_name='epoch'): # Evaluate on each set separately for s in self.set_name: # Apply model predictions if self.beam_search: params_prediction = { 'batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars['n_parallel_loaders'], 'predict_on_sets': [s], 'pos_unk': False, 'heuristic': 0, 'mapping': None } params_prediction.update( checkDefaultParamsBeamSearch(self.extra_vars)) predictions = self.model_to_eval.predictBeamSearchNet( self.ds, params_prediction)[s] else: orig_size = self.extra_vars.get('eval_orig_size', False) params_prediction = { 'batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars['n_parallel_loaders'], 'predict_on_sets': [s] } # Convert predictions postprocess_fun = None if self.is_3DLabel: postprocess_fun = [ self.ds.convert_3DLabels_to_bboxes, self.extra_vars[s]['references_orig_sizes'] ] elif orig_size: postprocess_fun = [ self.ds.resize_semantic_output, self.extra_vars[s]['eval_orig_size_id'] ] predictions = \ self.model_to_eval.predictNet(self.ds, params_prediction, postprocess_fun=postprocess_fun)[s] if self.is_text: if params_prediction.get('pos_unk', False): samples = predictions[0] alphas = predictions[1] if eval('self.ds.loaded_raw_' + s + '[0]'): sources = predictions[2] else: sources = [] for preds in predictions[2]: for src in preds[self.input_text_id]: sources.append(src) sources = decode_predictions_beam_search( sources, self.index2word_x, pad_sequences=True, verbose=self.verbose) heuristic = params_prediction['heuristic'] else: samples = predictions alphas = None heuristic = None sources = None if self.out_pred_idx is not None: samples = samples[self.out_pred_idx] # Convert predictions into sentences if self.beam_search: predictions = decode_predictions_beam_search( samples, self.index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=params_prediction['mapping'], verbose=self.verbose) else: predictions = decode_predictions( predictions, 1, # always set temperature to 1 self.index2word_y, self.sampling_type, verbose=self.verbose) # Store predictions if self.write_samples: # Store result filepath = self.save_path + '/' + s + '_' + counter_name + '_' + str( epoch) + '.pred' # results file if self.write_type == 'list': list2file(filepath, predictions) elif self.write_type == 'vqa': list2vqa(filepath, predictions, self.extra_vars[s]['question_ids']) elif self.write_type == 'listoflists': listoflists2file(filepath, predictions) elif self.write_type == 'numpy': numpy2file(filepath, predictions) elif self.write_type == '3DLabels': # TODO: print("WRITE SAMPLES FUNCTION NOT IMPLEMENTED") else: raise NotImplementedError('The store type "' + self.write_type + '" is not implemented.') # Evaluate on each metric for metric in self.metric_name: if self.verbose > 0: logging.info('Evaluating on metric ' + metric) filepath = self.save_path + '/' + s + '.' + metric # results file # Evaluate on the chosen metric metrics = evaluation.select[metric](pred_list=predictions, verbose=self.verbose, extra_vars=self.extra_vars, split=s) # Print results to file and store in model log with open(filepath, 'a') as f: header = counter_name + ',' line = str(epoch) + ',' # Store in model log self.model_to_eval.log(s, counter_name, epoch) for metric_ in sorted(metrics): value = metrics[metric_] header += metric_ + ', ' line += str(value) + ', ' # Store in model log self.model_to_eval.log(s, metric_, value) if epoch == 1 or epoch == self.start_eval_on_epoch: f.write(header + '\n') f.write(line + '\n') if self.verbose > 0: logging.info('Done evaluating on metric ' + metric) # Save the model if self.save_each_evaluation: from keras_wrapper.cnn_model import saveModel saveModel(self.model_to_eval, epoch, store_iter=not self.eval_on_epochs)
def get_model_predictions(asts_path): print("os.getcwd()", os.getcwd()) cur_dir = os.path.dirname(os.path.abspath(__file__)) print("cur_dir", cur_dir) # if not os.path.isdir(os.path.join(os.getcwd(), 'keras')): # print(subprocess.run(f'git clone https://github.com/MarcBS/keras.git', shell=True, # stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)) # # nmt_keras_dir = os.path.join(os.getcwd, 'nmt-keras') # if not os.path.isdir(os.path.join(os.getcwd(), 'nmt-keras')): # print(subprocess.run(f'git clone https://github.com/lvapeab/nmt-keras && cd "nmt-keras" && pipenv install -e .', shell=True, # stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)) # # print(subprocess.run(f'cd {nmt_keras_dir} && pipenv install -e .', shell=True, # # stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)) # print("ran cmds!!!") # sys.path.insert(0, os.path.join(os.getcwd(), 'nmt-keras')) # print("sys path!!!", sys.path) dataset = loadDataset( f'{cur_dir}/assets/epoch_{MODEL_EPOCH}_model_wrapper.pkl') with open('{cur_dir}/assets/params.json', 'r') as params_file: params = json.load(params_file) dataset.setInput(asts_path, 'test', type='text', id='source_text', pad_on_batch=True, tokenization=params['tokenize_x'], fill='end', max_text_len=params['x_max_text_len'], min_occ=0) dataset.setInput(None, 'test', type='ghost', id='state_below', required=False) dataset.setRawInput(asts_path, 'test', type='file-name', id='raw_source_text', overwrite_split=True) nmt_model = loadModel('{cur_dir}/assets', MODEL_EPOCH) prediction_params = get_prediction_params() predictions = nmt_model.predictBeamSearchNet(dataset, params_prediction)['test'] vocab = dataset.vocabulary['target_text']['idx2words'] samples = predictions['samples'] # Get word indices from the samples. predictions = decode_predictions_beam_search(samples, vocab, verbose=params['VERBOSE']) return predictions
def on_batch_end(self, n_update, logs=None): self.cum_update += 1 if self.epoch_count + self.reload_epoch < self.start_sampling_on_epoch: return elif self.cum_update % self.each_n_updates != 0: return # Evaluate on each set separately for s in self.set_name: if self.beam_search: params_prediction = {'max_batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars['n_parallel_loaders'], 'predict_on_sets': [s], 'n_samples': self.n_samples, 'pos_unk': False} params_prediction.update(checkDefaultParamsBeamSearch(self.extra_vars)) predictions, truths, sources = self.model_to_eval.predictBeamSearchNet(self.ds, params_prediction) else: params_prediction = {'batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars['n_parallel_loaders'], 'predict_on_sets': [s], 'n_samples': self.n_samples, 'verbose': self.verbose, } # Convert predictions postprocess_fun = None if self.is_3DLabel: postprocess_fun = [self.ds.convert_3DLabels_to_bboxes, self.extra_vars[s]['references_orig_sizes']] predictions = self.model_to_eval.predictNet(self.ds, params_prediction, postprocess_fun=postprocess_fun) if self.print_sources: if self.in_pred_idx is not None: sources = [srcs[self.in_pred_idx][0] for srcs in sources] sources = decode_predictions_beam_search(sources, self.index2word_x, pad_sequences=True, verbose=self.verbose) if s in predictions: if params_prediction['pos_unk']: samples = predictions[s][0] alphas = predictions[s][1] heuristic = self.extra_vars['heuristic'] else: samples = predictions[s] alphas = None heuristic = None predictions = predictions[s] if self.is_text: if self.out_pred_idx is not None: samples = samples[self.out_pred_idx] # Convert predictions into sentences if self.beam_search: predictions = decode_predictions_beam_search(samples, self.index2word_y, glossary=self.extra_vars.get('glossary', None), alphas=alphas, x_text=sources, heuristic=heuristic, mapping=self.extra_vars.get('mapping', None), verbose=self.verbose) else: predictions = decode_predictions(samples, 1, self.index2word_y, self.sampling_type, verbose=self.verbose) truths = decode_predictions_one_hot(truths, self.index2word_y, verbose=self.verbose) # Apply detokenization function if needed if self.extra_vars.get('apply_detokenization', False): if self.print_sources: sources = list(map(self.extra_vars['detokenize_f'], sources)) predictions = list(map(self.extra_vars['detokenize_f'], predictions)) truths = list(map(self.extra_vars['detokenize_f'], truths)) # Write samples if self.print_sources: # Write samples for i, (source, sample, truth) in list(enumerate(zip(sources, predictions, truths))): if sys.version_info.major == 2: source = str(source.encode('utf-8')) sample = str(sample.encode('utf-8')) truth = str(truth.encode('utf-8')) print("Source (%d): %s" % (i, source)) print("Hypothesis (%d): %s" % (i, sample)) print("Reference (%d): %s" % (i, truth)) print("") else: for i, (sample, truth) in list(enumerate(zip(predictions, truths))): if sys.version_info.major == 2: sample = str(sample.encode('utf-8')) truth = str(truth.encode('utf-8')) print("Hypothesis (%d): %s" % (i, sample)) print("Reference (%d): %s" % (i, truth)) print("")
def evaluate(self, epoch, counter_name='epoch', logs=None): if logs is None: logs = {} # Change inputs and outputs mappings for evaluation self.changeInOutMappings() # Evaluate on each set separately all_metrics = [] for s in self.set_name: # Apply model predictions if self.beam_search: params_prediction = {'max_batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars.get('n_parallel_loaders', 1), 'predict_on_sets': [s], 'beam_batch_size': self.beam_batch_size if self.beam_batch_size is not None else self.batch_size, 'pos_unk': False, 'normalize': self.normalize, 'normalization_type': self.normalization_type, 'max_eval_samples': self.max_eval_samples } params_prediction.update(checkDefaultParamsBeamSearch(self.extra_vars)) predictions_all = self.model_to_eval.predictBeamSearchNet(self.ds, params_prediction)[s] else: orig_size = self.extra_vars.get('eval_orig_size', False) params_prediction = {'batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars.get('n_parallel_loaders', 1), 'predict_on_sets': [s], 'normalize': self.normalize, 'normalization_type': self.normalization_type, 'max_eval_samples': self.max_eval_samples, 'model_name': self.model_name, } # Convert predictions postprocess_fun = None if self.is_3DLabel: postprocess_fun = [self.ds.convert_3DLabels_to_bboxes, self.extra_vars[s]['references_orig_sizes']] elif orig_size: postprocess_fun = [self.ds.resize_semantic_output, self.extra_vars[s]['eval_orig_size_id']] predictions_all = \ self.model_to_eval.predictNet(self.ds, params_prediction, postprocess_fun=postprocess_fun)[s] # Single-output model if not self.gt_pos or self.gt_pos == 0 or len(self.gt_pos) == 1: if len(predictions_all) != 2: predictions_all = [predictions_all] gt_positions = [0] # Multi-output model else: gt_positions = self.gt_pos # Select each output to evaluate separately for gt_pos, type_out, these_metrics, gt_id, write_type, index2word_y, index2word_x in zip( gt_positions, self.output_types, self.metric_name, self.gt_id, self.write_type, self.index2word_y, self.index2word_x): predictions = predictions_all[gt_pos] if self.verbose > 0: print('') logging.info('Prediction output ' + str(gt_pos) + ': ' + str( gt_id) + ' (' + str(type_out) + ')') # Postprocess outputs of type text if type_out == 'text': if params_prediction.get('pos_unk', False): samples = predictions[0] alphas = predictions[1] if eval('self.ds.loaded_raw_' + s + '[0]'): sources = predictions[2] else: sources = [] for preds in predictions[2]: for src in preds[self.input_text_id]: sources.append(src) sources = decode_predictions_beam_search(sources, index2word_x, pad_sequences=True, verbose=self.verbose) heuristic = self.extra_vars['heuristic'] else: samples = predictions alphas = None heuristic = None sources = None if self.out_pred_idx is not None: samples = samples[self.out_pred_idx] # Convert predictions into sentences if self.beam_search: predictions = decode_predictions_beam_search(samples, index2word_y, glossary=self.extra_vars.get('glossary', None), alphas=alphas, x_text=sources, heuristic=heuristic, mapping=self.extra_vars.get('mapping', None), verbose=self.verbose) else: probs = predictions predictions = decode_predictions(predictions, 1, # always set temperature to 1 index2word_y, self.sampling_type, verbose=self.verbose) # Apply detokenization function if needed if self.extra_vars.get('apply_detokenization', False): predictions = list(map(self.extra_vars['detokenize_f'], predictions)) # Postprocess outputs of type binary elif type_out == 'binary': predictions = decode_multilabel(predictions, index2word_y, min_val=self.min_pred_multilabel[ gt_pos], verbose=self.verbose) # Prepare references # exec ("y_raw = self.ds.Y_" + s + "[gt_id]") y_split = getattr(self.ds, 'Y_' + s) y_raw = y_split[gt_id] self.extra_vars[gt_pos][s]['references'] = self.ds.loadBinary(y_raw, gt_id) # Postprocess outputs of type 3DLabel elif type_out == '3DLabel': self.extra_vars[gt_pos][s] = dict() # exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]') y_split = getattr(self.ds, 'Y_' + s) ref = y_split[gt_id] [ref, original_sizes] = self.ds.convert_GT_3DLabels_to_bboxes( ref) self.extra_vars[gt_pos][s]['references'] = ref self.extra_vars[gt_pos][s]['references_orig_sizes'] = original_sizes # Postprocess outputs of type 3DSemanticLabel elif type_out == '3DSemanticLabel': self.extra_vars[gt_pos]['eval_orig_size'] = self.eval_orig_size self.extra_vars[gt_pos][s] = dict() # exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]') y_split = getattr(self.ds, 'Y_' + s) ref = y_split[gt_id] if self.eval_orig_size: old_crop = copy.deepcopy(self.ds.img_size_crop) self.ds.img_size_crop = copy.deepcopy(self.ds.img_size) self.extra_vars[gt_pos][s]['eval_orig_size_id'] = np.array([gt_id] * len(ref)) ref = self.ds.load_GT_3DSemanticLabels(ref, gt_id) if self.eval_orig_size: self.ds.img_size_crop = copy.deepcopy(old_crop) self.extra_vars[gt_pos][s]['references'] = ref # Other output data types else: # exec ("self.extra_vars[gt_pos][s]['references'] = self.ds.Y_" + s + "[gt_id]") y_split = getattr(self.ds, 'Y_' + s) self.extra_vars[gt_pos][s]['references'] = y_split[gt_id] # Store predictions if self.write_samples: # Store result filepath = self.save_path + '/' + s + '_' + counter_name + '_' + str(epoch) + '_output_' + str(gt_pos) + '.pred' # results file if write_type == 'list': list2file(filepath, predictions) elif write_type == 'vqa': try: # exec ('refs = self.ds.Y_' + s + '[gt_id]') y_split = getattr(self.ds, 'Y_' + s) refs = y_split[gt_id] except Exception: refs = ['N/A' for _ in range(probs.shape[0])] extra_data_plot = {'reference': refs, 'probs': probs, 'vocab': index2word_y} list2vqa(filepath, predictions, self.extra_vars[gt_pos][s]['question_ids'], extra=extra_data_plot) elif write_type == 'listoflists': listoflists2file(filepath, predictions) elif write_type == 'numpy': numpy2file(filepath, predictions) elif write_type == '3DLabels': raise NotImplementedError( 'Write 3DLabels function is not implemented') elif write_type == '3DSemanticLabel': folder_path = self.save_path + '/' + s + '_' + counter_name + '_' + str( epoch) # results folder numpy2imgs(folder_path, predictions, eval('self.ds.X_' + s + '["' + self.input_id + '"]'), self.ds) else: raise NotImplementedError('The store type "' + self.write_type + '" is not implemented.') # Evaluate on each metric for metric in these_metrics: if self.verbose > 0: logging.info('Evaluating on metric ' + metric) filepath = self.save_path + '/' + s + '.' + metric # results file if s == 'train': logging.info( "WARNING: evaluation results on 'train' split might be incorrect when" "applying random image shuffling.") # Evaluate on the chosen metric metrics = evaluation.select[metric]( pred_list=predictions, verbose=self.verbose, extra_vars=self.extra_vars[gt_pos], split=s) # Print results to file and store in model log with open(filepath, 'a') as f: header = counter_name + ',' line = str(epoch) + ',' # Store in model log self.model_to_eval.log(s, counter_name, epoch) for metric_ in sorted(metrics): value = metrics[metric_] # Multiple-output model if self.gt_pos and self.gt_pos != 0: metric_ += '_output_' + str(gt_pos) all_metrics.append(metric_) header += metric_ + ',' line += str(value) + ',' # Store in model log self.model_to_eval.log(s, metric_, value) if not self.written_header: f.write(header + '\n') self.written_header = True f.write(line + '\n') if self.verbose > 0: logging.info('Done evaluating on metric ' + metric) # Store losses if logs.get('loss') is not None: self.model_to_eval.log('train', 'train_loss', logs['loss']) if logs.get('valid_loss') is not None: self.model_to_eval.log('val', 'val_loss', logs['valid_loss']) # Plot results so far if self.do_plot: if self.metric_name: self.model_to_eval.plot(counter_name, set(all_metrics), self.set_name, upperbound=self.max_plot) # Save the model if self.save_each_evaluation: from keras_wrapper.cnn_model import saveModel saveModel(self.model_to_eval, epoch, store_iter=not self.eval_on_epochs) # Recover inputs and outputs mappings for resume training self.recoverInOutMappings()
def user_input_prediction(best_epoch): params = load_parameters() dataset = loadDataset(PATH + "dataset/Dataset_tutorial_dataset.pkl") params['INPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len[ params['INPUTS_IDS_DATASET'][0]] params['OUTPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len[ params['OUTPUTS_IDS_DATASET'][0]] # Load model nmt_model = loadModel(PATH + 'model/', best_epoch) params_prediction = { 'language': 'en', 'tokenize_f': eval('dataset.' + 'tokenize_basic'), 'beam_size': 12, 'length_penalty': True, 'length_norm_factor': 1.0, 'optimized_search': True, 'model_inputs': params['INPUTS_IDS_MODEL'], 'model_outputs': params['OUTPUTS_IDS_MODEL'], 'dataset_inputs': params['INPUTS_IDS_DATASET'], 'dataset_outputs': params['OUTPUTS_IDS_DATASET'], 'n_parallel_loaders': 1, 'maxlen': 50, 'model_inputs': ['source_text', 'state_below'], 'model_outputs': ['target_text'], 'dataset_inputs': ['source_text', 'state_below'], 'dataset_outputs': ['target_text'], 'normalize': True, 'pos_unk': True, 'heuristic': 0, 'state_below_maxlen': 1, 'predict_on_sets': ['test'], 'verbose': 0, } result = pyfiglet.figlet_format("TESTING WITH USER INPUT".format(mode), font="digital") print(result) while True: print("Input a sentence:") user_input = input() with open('user_input.txt', 'w') as f: f.write(user_input) dataset.setInput('user_input.txt', 'test', type='text', id='source_text', pad_on_batch=True, tokenization='tokenize_basic', fill='end', max_text_len=100, min_occ=1, overwrite_split=True) dataset.setInput(None, 'test', type='ghost', id='state_below', required=False, overwrite_split=True) dataset.setRawInput('user_input.txt', 'test', type='file-name', id='raw_source_text', overwrite_split=True) vocab = dataset.vocabulary['target_text']['idx2words'] predictions = nmt_model.predictBeamSearchNet(dataset, params_prediction)['test'] predictions = decode_predictions_beam_search( predictions[ 0], # The first element of predictions contain the word indices. vocab, verbose=params['VERBOSE']) print(predictions[0])
def apply_Video_model(params): """ Function for using a previously trained model for sampling. """ ########### Load data dataset = build_dataset(params) if not '-vidtext-embed' in params['DATASET_NAME']: params['OUTPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len[ params['OUTPUTS_IDS_DATASET'][0]] else: params['OUTPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len[ params['INPUTS_IDS_DATASET'][1]] ########### ########### Load model video_model = loadModel(params['STORE_PATH'], params['SAMPLING_RELOAD_POINT'], reload_epoch=params['SAMPLING_RELOAD_EPOCH']) video_model.setOptimizer() ########### ########### Apply sampling extra_vars = dict() extra_vars['tokenize_f'] = eval('dataset.' + params['TOKENIZATION_METHOD']) extra_vars['language'] = params.get('TRG_LAN', 'en') for s in params["EVAL_ON_SETS"]: # Apply model predictions params_prediction = { 'max_batch_size': params['BATCH_SIZE'], 'n_parallel_loaders': params['PARALLEL_LOADERS'], 'predict_on_sets': [s] } # Convert predictions into sentences if not '-vidtext-embed' in params['DATASET_NAME']: vocab = dataset.vocabulary[params['OUTPUTS_IDS_DATASET'] [0]]['idx2words'] else: vocab = None if params['BEAM_SEARCH']: params_prediction['beam_size'] = params['BEAM_SIZE'] params_prediction['maxlen'] = params['MAX_OUTPUT_TEXT_LEN_TEST'] params_prediction['optimized_search'] = params[ 'OPTIMIZED_SEARCH'] and '-upperbound' not in params[ 'DATASET_NAME'] params_prediction['model_inputs'] = params['INPUTS_IDS_MODEL'] params_prediction['model_outputs'] = params['OUTPUTS_IDS_MODEL'] params_prediction['dataset_inputs'] = params['INPUTS_IDS_DATASET'] params_prediction['dataset_outputs'] = params[ 'OUTPUTS_IDS_DATASET'] params_prediction['normalize_probs'] = params['NORMALIZE_SAMPLING'] params_prediction['alpha_factor'] = params['ALPHA_FACTOR'] params_prediction['temporally_linked'] = '-linked' in params['DATASET_NAME'] and '-upperbound' not in \ params[ 'DATASET_NAME'] and '-video' not in \ params[ 'DATASET_NAME'] predictions = video_model.predictBeamSearchNet( dataset, params_prediction)[s] predictions = decode_predictions_beam_search( predictions, vocab, verbose=params['VERBOSE']) else: predictions = video_model.predictNet(dataset, params_prediction)[s] predictions = decode_predictions(predictions, 1, vocab, params['SAMPLING'], verbose=params['VERBOSE']) # Store result filepath = video_model.model_path + '/' + s + '_sampling.pred' # results file if params['SAMPLING_SAVE_MODE'] == 'list': list2file(filepath, predictions) else: raise Exception, 'Only "list" is allowed in "SAMPLING_SAVE_MODE"' # Evaluate if any metric in params['METRICS'] for metric in params['METRICS']: logging.info('Evaluating on metric ' + metric) filepath = video_model.model_path + '/' + s + '_sampling.' + metric # results file # Evaluate on the chosen metric extra_vars[s] = dict() extra_vars[s]['references'] = dataset.extra_variables[s][ params['OUTPUTS_IDS_DATASET'][0]] metrics = selectMetric[metric](pred_list=predictions, verbose=1, extra_vars=extra_vars, split=s) # Print results to file with open(filepath, 'w') as f: header = '' line = '' for metric_ in sorted(metrics): value = metrics[metric_] header += metric_ + ',' line += str(value) + ',' f.write(header + '\n') f.write(line + '\n') logging.info('Done evaluating on metric ' + metric)
def interactive_simulation(): args = parse_args() # Update parameters if args.config is not None: logger.info('Reading parameters from %s.' % args.config) params = update_parameters({}, pkl2dict(args.config)) else: logger.info('Reading parameters from config.py.') params = load_parameters() if args.online: from config_online import load_parameters as load_parameters_online online_parameters = load_parameters_online(params) params = update_parameters(params, online_parameters) try: for arg in args.changes: try: k, v = arg.split('=') except ValueError: print( 'Overwritten arguments must have the form key=Value. \n Currently are: %s' % str(args.changes)) exit(1) try: params[k] = ast.literal_eval(v) except ValueError: params[k] = v except ValueError: print('Error processing arguments: (', k, ",", v, ")") exit(2) check_params(params) if args.verbose: logging.info("params = " + str(params)) dataset = loadDataset(args.dataset) # dataset = update_dataset_from_file(dataset, args.source, params, splits=args.splits, remove_outputs=True) # Dataset backwards compatibility bpe_separator = dataset.BPE_separator if hasattr( dataset, "BPE_separator") and dataset.BPE_separator is not None else u'@@' # Set tokenization method params[ 'TOKENIZATION_METHOD'] = 'tokenize_bpe' if args.tokenize_bpe else params.get( 'TOKENIZATION_METHOD', 'tokenize_none') # Build BPE tokenizer if necessary if 'bpe' in params['TOKENIZATION_METHOD'].lower(): logger.info('Building BPE') if not dataset.BPE_built: dataset.build_bpe(params.get( 'BPE_CODES_PATH', params['DATA_ROOT_PATH'] + '/training_codes.joint'), separator=bpe_separator) # Build tokenization function tokenize_f = eval('dataset.' + params.get('TOKENIZATION_METHOD', 'tokenize_none')) if args.online: # Traning params params_training = { # Traning params 'n_epochs': params['MAX_EPOCH'], 'shuffle': False, 'loss': params.get('LOSS', 'categorical_crossentropy'), 'batch_size': params.get('BATCH_SIZE', 1), 'homogeneous_batches': False, 'optimizer': params.get('OPTIMIZER', 'SGD'), 'lr': params.get('LR', 0.1), 'lr_decay': params.get('LR_DECAY', None), 'lr_gamma': params.get('LR_GAMMA', 1.), 'epochs_for_save': -1, 'verbose': args.verbose, 'eval_on_sets': params['EVAL_ON_SETS_KERAS'], 'n_parallel_loaders': params['PARALLEL_LOADERS'], 'extra_callbacks': [], # callbacks, 'reload_epoch': 0, 'epoch_offset': 0, 'data_augmentation': params['DATA_AUGMENTATION'], 'patience': params.get('PATIENCE', 0), 'metric_check': params.get('STOP_METRIC', None), 'eval_on_epochs': params.get('EVAL_EACH_EPOCHS', True), 'each_n_epochs': params.get('EVAL_EACH', 1), 'start_eval_on_epoch': params.get('START_EVAL_ON_EPOCH', 0), 'additional_training_settings': { 'k': params.get('K', 1), 'tau': params.get('TAU', 1), 'lambda': params.get('LAMBDA', 0.5), 'c': params.get('C', 0.5), 'd': params.get('D', 0.5) } } else: params_training = dict() params['OUTPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len[ params['OUTPUTS_IDS_DATASET'][0]] logger.info("<<< Using an ensemble of %d models >>>" % len(args.models)) if args.online: # Load trainable model(s) logging.info('Loading models from %s' % str(args.models)) model_instances = [ Captioning_Model(params, model_type=params['MODEL_TYPE'], verbose=params['VERBOSE'], model_name=params['MODEL_NAME'] + '_' + str(i), vocabularies=dataset.vocabulary, store_path=params['STORE_PATH'], clear_dirs=False, set_optimizer=False) for i in range(len(args.models)) ] models = [ updateModel(model, path, -1, full_path=True) for (model, path) in zip(model_instances, args.models) ] # Set additional inputs to models if using a custom loss function params['USE_CUSTOM_LOSS'] = True if 'PAS' in params[ 'OPTIMIZER'] else False if params['N_BEST_OPTIMIZER']: logging.info('Using N-best optimizer') models = build_online_models(models, params) online_trainer = OnlineTrainer(models, dataset, None, None, params_training, verbose=args.verbose) else: # Otherwise, load regular model(s) models = [loadModel(m, -1, full_path=True) for m in args.models] # Load text files logger.info("<<< Storing corrected hypotheses into: %s >>>" % str(args.dest)) ftrans = open(args.dest, 'w') ftrans.close() # Do we want to save the original sentences? if args.original_dest is not None: logger.info("<<< Storing original hypotheses into: %s >>>" % str(args.original_dest)) ftrans_ori = open(args.original_dest, 'w') ftrans_ori.close() if args.references is not None: ftrg = codecs.open(args.references, 'r', encoding='utf-8' ) # File with post-edited (or reference) sentences. all_references = ftrg.read().split('\n') if all_references[-1] == u'': all_references = all_references[:-1] # Get word2index and index2word dictionaries index2word_y = dataset.vocabulary[params['OUTPUTS_IDS_DATASET'] [0]]['idx2words'] word2index_y = dataset.vocabulary[params['OUTPUTS_IDS_DATASET'] [0]]['words2idx'] unk_id = dataset.extra_words['<unk>'] # Initialize counters total_errors = 0 total_words = 0 total_chars = 0 total_mouse_actions = 0 try: for s in args.splits: # Apply model predictions params_prediction = { 'max_batch_size': params['BATCH_SIZE'], 'n_parallel_loaders': params['PARALLEL_LOADERS'], 'predict_on_sets': [s], 'beam_size': params['BEAM_SIZE'], 'maxlen': params['MAX_OUTPUT_TEXT_LEN_TEST'], 'optimized_search': params['OPTIMIZED_SEARCH'], 'model_inputs': params['INPUTS_IDS_MODEL'], 'model_outputs': params['OUTPUTS_IDS_MODEL'], 'dataset_inputs': params['INPUTS_IDS_DATASET'], 'dataset_outputs': params['OUTPUTS_IDS_DATASET'], 'normalize_probs': params.get('NORMALIZE_SAMPLING', False), 'alpha_factor': params.get('ALPHA_FACTOR', 1.0), 'normalize': params.get('NORMALIZATION', False), 'normalization_type': params.get('NORMALIZATION_TYPE', None), 'data_augmentation': params.get('DATA_AUGMENTATION', False), 'mean_substraction': params.get('MEAN_SUBTRACTION', False), 'wo_da_patch_type': params.get('WO_DA_PATCH_TYPE', 'whole'), 'da_patch_type': params.get('DA_PATCH_TYPE', 'resize_and_rndcrop'), 'da_enhance_list': params.get('DA_ENHANCE_LIST', None), 'heuristic': params.get('HEURISTIC', None), 'search_pruning': params.get('SEARCH_PRUNING', False), 'state_below_index': -1, 'output_text_index': 0, 'apply_tokenization': params.get('APPLY_TOKENIZATION', False), 'tokenize_f': eval('dataset.' + params.get('TOKENIZATION_METHOD', 'tokenize_none')), 'apply_detokenization': params.get('APPLY_DETOKENIZATION', True), 'detokenize_f': eval('dataset.' + params.get('DETOKENIZATION_METHOD', 'detokenize_none')), 'coverage_penalty': params.get('COVERAGE_PENALTY', False), 'length_penalty': params.get('LENGTH_PENALTY', False), 'length_norm_factor': params.get('LENGTH_NORM_FACTOR', 0.0), 'coverage_norm_factor': params.get('COVERAGE_NORM_FACTOR', 0.0), 'pos_unk': False, 'state_below_maxlen': -1 if params.get('PAD_ON_BATCH', True) else params.get( 'MAX_OUTPUT_TEXT_LEN_TEST', 50), 'output_max_length_depending_on_x': params.get('MAXLEN_GIVEN_X', False), 'output_max_length_depending_on_x_factor': params.get('MAXLEN_GIVEN_X_FACTOR', 3), 'output_min_length_depending_on_x': params.get('MINLEN_GIVEN_X', False), 'output_min_length_depending_on_x_factor': params.get('MINLEN_GIVEN_X_FACTOR', 2), 'attend_on_output': params.get('ATTEND_ON_OUTPUT', 'transformer' in params['MODEL_TYPE'].lower()), 'n_best_optimizer': params.get('N_BEST_OPTIMIZER', False) } # Build interactive sampler interactive_beam_searcher = InteractiveBeamSearchSampler( models, dataset, params_prediction, excluded_words=None, verbose=args.verbose) start_time = time.time() if args.verbose: logging.info("Params prediction = " + str(params_prediction)) if args.online: logging.info("Params training = " + str(params_training)) n_samples = getattr(dataset, 'len_' + s) if args.references is None: all_references = dataset.extra_variables[s][ params['OUTPUTS_IDS_DATASET'][0]] # Start to translate the source file interactively for n_sample in range(n_samples): errors_sentence = 0 mouse_actions_sentence = 0 hypothesis_number = 0 # Load data from dataset current_input = dataset.getX_FromIndices( s, [n_sample], normalization_type=params_prediction.get( 'normalization_type'), normalization=params_prediction.get('normalize', False), dataAugmentation=params_prediction.get( 'data_augmentation', False), wo_da_patch_type=params_prediction.get( 'wo_da_patch_type', 'whole'), da_patch_type=params_prediction.get( 'da_patch_type', 'resize_and_rndcrop'), da_enhance_list=params_prediction.get( 'da_enhance_list', None))[0][0] # Load references references = all_references[n_sample] tokenized_references = list(map( tokenize_f, references)) if args.tokenize_references else references # Get reference as desired by the user, i.e. detokenized if necessary reference = list(map(params_prediction['detokenize_f'], tokenized_references)) if \ args.detokenize_bpe else tokenized_references # Detokenize line for nicer logging :) logger.debug(u'\n\nProcessing sample %d' % (n_sample + 1)) logger.debug(u'Target: %s' % reference) # 1. Get a first hypothesis trans_indices, costs, alphas = interactive_beam_searcher.sample_beam_search_interactive( current_input) # 1.2 Decode hypothesis hypothesis = decode_predictions_beam_search([trans_indices], index2word_y, pad_sequences=True, verbose=0)[0] # 1.3 Store result (optional) hypothesis = params_prediction['detokenize_f'](hypothesis) \ if params_prediction.get('apply_detokenization', False) else hypothesis if args.original_dest is not None: if params['SAMPLING_SAVE_MODE'] == 'list': list2file(args.original_dest, [hypothesis], permission='a') else: raise Exception( 'Only "list" is allowed in "SAMPLING_SAVE_MODE"') logger.debug(u'Hypo_%d: %s' % (hypothesis_number, hypothesis)) # 2.0 Interactive translation if hypothesis in tokenized_references: # 2.1 If the sentence is correct, we validate it pass else: # 2.2 Wrong hypothesis -> Interactively translate the sentence correct_hypothesis = False last_correct_pos = 0 while not correct_hypothesis: # 2.2.1 Empty data structures for the next sentence fixed_words_user = OrderedDict() unk_words_dict = OrderedDict() isle_indices = [] unks_in_isles = [] if args.prefix: # 2.2.2 Compute longest common character prefix (LCCP) reference_idx, next_correction_pos, validated_prefix = common_prefixes( hypothesis, tokenized_references) else: # 2.2.2 Compute common character segments #TODO next_correction_pos, validated_prefix, validated_segments = common_segments( hypothesis, reference) reference = tokenized_references[reference_idx] if next_correction_pos == len(reference): correct_hypothesis = True break # 2.2.3 Get next correction by checking against the reference next_correction = reference[next_correction_pos] # 2.2.4 Tokenize the prefix properly (possibly applying BPE) tokenized_validated_prefix = tokenize_f( validated_prefix + next_correction) # 2.2.5 Validate words for pos, word in enumerate( tokenized_validated_prefix.split()): fixed_words_user[pos] = word2index_y.get( word, unk_id) if word2index_y.get(word) is None: unk_words_dict[pos] = word # 2.2.6 Constrain search for the last word last_user_word_pos = list(fixed_words_user.keys())[-1] if next_correction != u' ': last_user_word = tokenized_validated_prefix.split( )[-1] filtered_idx2word = dict( (word2index_y[candidate_word], candidate_word) for candidate_word in word2index_y if candidate_word[:len(last_user_word)] == last_user_word) if filtered_idx2word != dict(): del fixed_words_user[last_user_word_pos] if last_user_word_pos in unk_words_dict.keys(): del unk_words_dict[last_user_word_pos] else: filtered_idx2word = dict() logger.debug(u'"%s" to character %d.' % (next_correction, next_correction_pos)) # 2.2.7 Generate a hypothesis compatible with the feedback provided by the user hypothesis = generate_constrained_hypothesis( interactive_beam_searcher, current_input, fixed_words_user, params_prediction, args, isle_indices, filtered_idx2word, index2word_y, None, None, None, unk_words_dict.keys(), unk_words_dict.values(), unks_in_isles) hypothesis_number += 1 hypothesis = u' '.join( hypothesis) # Hypothesis is unicode hypothesis = params_prediction['detokenize_f'](hypothesis) \ if args.detokenize_bpe else hypothesis logger.debug(u'Target: %s' % reference) logger.debug(u"Hypo_%d: %s" % (hypothesis_number, hypothesis)) # 2.2.8 Add a keystroke errors_sentence += 1 # 2.2.9 Add a mouse action if we moved the pointer if next_correction_pos - last_correct_pos > 1: mouse_actions_sentence += 1 last_correct_pos = next_correction_pos # 2.3 Final check: The reference is a subset of the hypothesis: Cut the hypothesis if len(reference) < len(hypothesis): hypothesis = hypothesis[:len(reference)] errors_sentence += 1 logger.debug(u"Cutting hypothesis") # 2.4 Security assertion assert hypothesis in references, "Error: The final hypothesis does not match with the reference! \n" \ "\t Split: %s \n" \ "\t Sentence: %d \n" \ "\t Hypothesis: %s\n" \ "\t Reference: %s" % (s, n_sample + 1, hypothesis, reference) # 3. Update user effort counters mouse_actions_sentence += 1 # This +1 is the validation action chars_sentence = len(hypothesis) total_errors += errors_sentence total_words += len(hypothesis.split()) total_chars += chars_sentence total_mouse_actions += mouse_actions_sentence # 3.1 Log some info logger.debug(u"Final hypotesis: %s" % hypothesis) logger.debug( u"%d errors. " u"Sentence WSR: %4f. " u"Sentence mouse strokes: %d " u"Sentence MAR: %4f. " u"Sentence MAR_c: %4f. " u"Sentence KSMR: %4f. " u"Accumulated (should only be considered for debugging purposes!) " u"WSR: %4f. " u"MAR: %4f. " u"MAR_c: %4f. " u"KSMR: %4f.\n\n\n\n" % (errors_sentence, float(errors_sentence) / len(hypothesis), mouse_actions_sentence, float(mouse_actions_sentence) / len(hypothesis), float(mouse_actions_sentence) / chars_sentence, float(errors_sentence + mouse_actions_sentence) / chars_sentence, float(total_errors) / total_words, float(total_mouse_actions) / total_words, float(total_mouse_actions) / total_chars, float(total_errors + total_mouse_actions) / total_chars)) # 4. If we are performing OL after each correct sample: if args.online: # 4.1 Compute model inputs # 4.1.1 Source text -> Already computed (used for the INMT process) # 4.1.2 State below state_below = dataset.loadText( [reference], vocabularies=dataset.vocabulary[ params['OUTPUTS_IDS_DATASET'][0]], max_len=params['MAX_OUTPUT_TEXT_LEN_TEST'], offset=1, fill=dataset.fill_text[params['INPUTS_IDS_DATASET'] [-1]], pad_on_batch=dataset.pad_on_batch[ params['INPUTS_IDS_DATASET'][-1]], words_so_far=False, loading_X=True)[0] # 4.1.3 Ground truth sample -> Interactively translated sentence trg_seq = dataset.loadTextOneHot( [reference], vocabularies=dataset.vocabulary[ params['OUTPUTS_IDS_DATASET'][0]], vocabulary_len=dataset.vocabulary_len[ params['OUTPUTS_IDS_DATASET'][0]], max_len=params['MAX_OUTPUT_TEXT_LEN_TEST'], offset=0, fill=dataset.fill_text[params['OUTPUTS_IDS_DATASET'] [0]], pad_on_batch=dataset.pad_on_batch[ params['OUTPUTS_IDS_DATASET'][0]], words_so_far=False, sample_weights=params['SAMPLE_WEIGHTS'], loading_X=False) # 4.2 Train online! online_trainer.train_online( [np.asarray([current_input]), state_below], trg_seq, trg_words=[reference]) # 5 Write correct sentences into a file list2file(args.dest, [hypothesis], permission='a') if (n_sample + 1) % 50 == 0: logger.info(u"%d sentences processed" % (n_sample + 1)) logger.info(u"Current speed is {} per sentence".format( (time.time() - start_time) / (n_sample + 1))) logger.info(u"Current WSR is: %f" % (float(total_errors) / total_words)) logger.info(u"Current MAR is: %f" % (float(total_mouse_actions) / total_words)) logger.info(u"Current MAR_c is: %f" % (float(total_mouse_actions) / total_chars)) logger.info(u"Current KSMR is: %f" % (float(total_errors + total_mouse_actions) / total_chars)) # 6. Final! # 6.1 Log some information print(u"Total number of errors:", total_errors) print(u"Total number selections", total_mouse_actions) print(u"WSR: %f" % (float(total_errors) / total_words)) print(u"MAR: %f" % (float(total_mouse_actions) / total_words)) print(u"MAR_c: %f" % (float(total_mouse_actions) / total_chars)) print(u"KSMR: %f" % (float(total_errors + total_mouse_actions) / total_chars)) except KeyboardInterrupt: print(u'Interrupted!') print(u"Total number of corrections (up to now):", total_errors) print(u"WSR: %f" % (float(total_errors) / total_words)) print(u"MAR: %f" % (float(total_mouse_actions) / total_words)) print(u"MAR_c: %f" % (float(total_mouse_actions) / total_chars)) print(u"KSMR: %f" % (float(total_errors + total_mouse_actions) / total_chars))
def generate_sample(self, source_sentence, validated_prefix=None, max_N=5, isle_indices=None, filtered_idx2word=None, unk_indices=None, unk_words=None): print ("In params prediction beam_size: ", self.params_prediction['beam_size']) logger.log(2, 'Beam size: %d' % (self.params_prediction['beam_size'])) generate_sample_start_time = time.time() if unk_indices is None: unk_indices = [] if unk_words is None: unk_words = [] tokenization_start_time = time.time() tokenized_input = self.general_tokenize_f(source_sentence, escape=False) tokenized_input = self.model_tokenize_f(tokenized_input) tokenization_end_time = time.time() logger.log(2, 'tokenization time: %.6f' % (tokenization_end_time - tokenization_start_time)) parse_input_start_time = time.time() src_seq, src_words = parse_input(tokenized_input, self.dataset, self.word2index_x) parse_input_end_time = time.time() logger.log(2, 'parse_input time: %.6f' % (parse_input_end_time - parse_input_start_time)) fixed_words_user = OrderedDict() unk_words_dict = OrderedDict() # If the user provided some feedback... if validated_prefix is not None: next_correction = validated_prefix[-1] if next_correction == self.eos_symbol: return validated_prefix[:-1].decode('utf-8') # 2.2.4 Tokenize the prefix properly (possibly applying BPE) # TODO: Here we are tokenizing the target language with the source language tokenizer prefix_tokenization_start_time = time.time() tokenized_validated_prefix = self.general_tokenize_f(validated_prefix, escape=False) tokenized_validated_prefix = self.model_tokenize_f(tokenized_validated_prefix) prefix_tokenization_end_time = time.time() logger.log(2, 'prefix_tokenization time: %.6f' % (prefix_tokenization_end_time - prefix_tokenization_start_time)) # 2.2.5 Validate words word_validation_start_time = time.time() for pos, word in enumerate(tokenized_validated_prefix.split()): fixed_words_user[pos] = self.word2index_y.get(word, self.unk_id) if self.word2index_y.get(word) is None: unk_words_dict[pos] = word word_validation_end_time = time.time() logger.log(2, 'word_validation time: %.6f' % (word_validation_end_time - word_validation_start_time)) # 2.2.6 Constrain search for the last word constrain_search_start_time = time.time() last_user_word_pos = list(fixed_words_user.keys())[-1] if next_correction != u' ': last_user_word = tokenized_validated_prefix.split()[-1] filtered_idx2word = dict((self.word2index_y[candidate_word], candidate_word) for candidate_word in self.word2index_y if candidate_word[:len(last_user_word)] == last_user_word) # if candidate_word.decode('utf-8')[:len(last_user_word)] == last_user_word) if filtered_idx2word != dict(): del fixed_words_user[last_user_word_pos] if last_user_word_pos in list(unk_words_dict.keys()): del unk_words_dict[last_user_word_pos] else: filtered_idx2word = dict() constrain_search_end_time = time.time() logger.log(2, 'constrain_search_end_time time: %.6f' % (constrain_search_end_time - constrain_search_start_time)) sample_beam_search_start_time = time.time() trans_indices, costs, alphas = \ self.interactive_beam_searcher.sample_beam_search_interactive(src_seq, fixed_words=copy.copy(fixed_words_user), max_N=max_N, isles=isle_indices, valid_next_words=filtered_idx2word, idx2word=self.index2word_y) sample_beam_search_end_time = time.time() logger.log(2, 'sample_beam_search time: %.6f' % (sample_beam_search_end_time - sample_beam_search_start_time)) # # Substitute possible unknown words in isles # unk_in_isles = [] # for isle_idx, isle_sequence, isle_words in unks_in_isles: # if unk_id in isle_sequence: # unk_in_isles.append((subfinder(isle_sequence, list(trans_indices)), isle_words)) if False and self.params_prediction['pos_unk']: alphas = [alphas] sources = [tokenized_input] heuristic = self.params_prediction['heuristic'] else: alphas = None heuristic = None sources = None # 1.2 Decode hypothesis decoding_predictions_start_time = time.time() hypothesis = decode_predictions_beam_search([trans_indices], self.index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=self.mapping, pad_sequences=True, verbose=0)[0] decoding_predictions_end_time = time.time() logger.log(2, 'decoding_predictions time: %.6f' % (decoding_predictions_end_time - decoding_predictions_start_time)) # for (words_idx, starting_pos), words in unk_in_isles: # for pos_unk_word, pos_hypothesis in enumerate(range(starting_pos, starting_pos + len(words_idx))): # hypothesis[pos_hypothesis] = words[pos_unk_word] # UNK words management unk_management_start_time = time.time() unk_indices = list(unk_words_dict) unk_words = list(unk_words_dict.values()) if len(unk_indices) > 0: # If we added some UNK word hypothesis = hypothesis.split() if len(hypothesis) < len(unk_indices): # The full hypothesis will be made up UNK words: for i, index in enumerate(range(0, len(hypothesis))): hypothesis[index] = unk_words[unk_indices[i]] for ii in range(i + 1, len(unk_words)): hypothesis.append(unk_words[ii]) else: # We put each unknown word in the corresponding gap for i, index in enumerate(unk_indices): if index < len(hypothesis): hypothesis[index] = unk_words[i] else: hypothesis.append(unk_words[i]) hypothesis = u' '.join(hypothesis) unk_management_end_time = time.time() logger.log(2, 'unk_management time: %.6f' % (unk_management_end_time - unk_management_start_time)) hypothesis_detokenization_start_time = time.time() hypothesis = self.model_detokenize_f(hypothesis) hypothesis = self.general_detokenize_f(hypothesis, unescape=False) hypothesis_detokenization_end_time = time.time() logger.log(2, 'hypothesis_detokenization time: %.6f' % (hypothesis_detokenization_end_time - hypothesis_detokenization_start_time)) generate_sample_end_time = time.time() logger.log(2, 'generate_sample time: %.6f' % (generate_sample_end_time - generate_sample_start_time)) return hypothesis
samples = predictions[0] alphas = predictions[1] sources = [ x.strip() for x in open(args.text, 'r').read().split('\n') ] sources = sources[:-1] if len(sources[-1]) == 0 else sources else: samples = predictions alphas = None heuristic = None sources = None predictions = decode_predictions_beam_search(samples, index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=mapping, verbose=args.verbose) # Apply detokenization function if needed if params.get('APPLY_DETOKENIZATION', False): predictions = map(detokenize_function, predictions) if args.n_best: n_best_predictions = [] i = 0 for i, (n_best_preds, n_best_scores, n_best_alphas) in enumerate(n_best): n_best_sample_score = [] for n_best_pred, n_best_score, n_best_alpha in zip( n_best_preds, n_best_scores, n_best_alphas):
def interactive_translation( src_seq, src_line, trg_line, params_prediction, args, tokenize_f, index2word_y, word2index_y, index2word_x, word2index_x, unk_id, total_errors, total_mouse_actions, n_line=-1, ): errors_sentence = 0 mouse_actions_sentence = 0 hypothesis_number = 0 # Get (tokenized) input tokenized_reference = tokenize_f( trg_line) if args.tokenize_references else trg_line # Get reference as desired by the user, i.e. detokenized if necessary reference = params_prediction['detokenize_f'](tokenized_reference) if \ args.detokenize_bpe else tokenized_reference # Detokenize line for nicer logging :) if args.detokenize_bpe: src_line = params_prediction['detokenize_f'](src_line) logger.debug(u'\n\nProcessing sentence %d' % n_line) logger.debug(u'Source: %s' % src_line) logger.debug(u'Target: %s' % reference) # 1. Get a first hypothesis trans_indices, costs, alphas = interactive_beam_searcher.sample_beam_search_interactive( src_seq) # 1.1 Set unk replacemet strategy if params_prediction['pos_unk']: alphas = [alphas] sources = [tokenized_input] heuristic = params_prediction['heuristic'] else: alphas = None heuristic = None sources = None # 1.2 Decode hypothesis hypothesis = decode_predictions_beam_search([trans_indices], index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=mapping, pad_sequences=True, verbose=0)[0] # 1.3 Store result (optional) hypothesis = params_prediction['detokenize_f'](hypothesis) \ if params_prediction.get('apply_detokenization', False) else hypothesis if args.original_dest is not None: filepath = args.original_dest # results file if params_prediction['SAMPLING_SAVE_MODE'] == 'list': list2file(filepath, [hypothesis + '\n'], permission='a') else: raise Exception('Only "list" is allowed in "SAMPLING_SAVE_MODE"') logger.debug(u'Hypo_%d: %s' % (hypothesis_number, hypothesis)) # 2.0 Interactive translation if hypothesis == reference: # 2.1 If the sentence is correct, we validate it pass else: # 2.2 Wrong hypothesis -> Interactively translate the sentence correct_hypothesis = False last_correct_pos = 0 while not correct_hypothesis: # 2.2.1 Empty data structures for the next sentence fixed_words_user = OrderedDict() unk_words_dict = OrderedDict() if not args.prefix: Exception( NotImplementedError, 'Segment-based interaction at' ' character level is still unimplemented') else: isle_indices = [] unks_in_isles = [] # 2.2.2 Compute longest common character prefix (LCCP) next_correction_pos, validated_prefix = common_prefix( hypothesis, reference) if next_correction_pos == len(reference): correct_hypothesis = True break # 2.2.3 Get next correction by checking against the reference next_correction = reference[next_correction_pos] # 2.2.4 Tokenize the prefix properly (possibly applying BPE) tokenized_validated_prefix = tokenize_f(validated_prefix + next_correction) # 2.2.5 Validate words for pos, word in enumerate(tokenized_validated_prefix.split()): fixed_words_user[pos] = word2index_y.get(word, unk_id) if word2index_y.get(word) is None: unk_words_dict[pos] = word # 2.2.6 Constrain search for the last word last_user_word_pos = fixed_words_user.keys()[-1] if next_correction != u' ': last_user_word = tokenized_validated_prefix.split()[-1] filtered_idx2word = dict( (word2index_y[candidate_word], candidate_word) for candidate_word in word2index_y if candidate_word.decode( 'utf-8')[:len(last_user_word)] == last_user_word) if filtered_idx2word != dict(): del fixed_words_user[last_user_word_pos] if last_user_word_pos in unk_words_dict.keys(): del unk_words_dict[last_user_word_pos] else: filtered_idx2word = dict() logger.debug(u'"%s" to character %d.' % (next_correction, next_correction_pos)) # 2.2.7 Generate a hypothesis compatible with the feedback provided by the user hypothesis = generate_constrained_hypothesis( interactive_beam_searcher, src_seq, fixed_words_user, params_prediction, args, isle_indices, filtered_idx2word, index2word_y, sources, heuristic, mapping, unk_words_dict.keys(), unk_words_dict.values(), unks_in_isles) hypothesis_number += 1 hypothesis = u' '.join(hypothesis) # Hypothesis is unicode hypothesis = params_prediction['detokenize_f'](hypothesis) \ if args.detokenize_bpe else hypothesis logger.debug(u'Target: %s' % reference) logger.debug(u"Hypo_%d: %s" % (hypothesis_number, hypothesis)) # 2.2.8 Add a keystroke errors_sentence += 1 # 2.2.9 Add a mouse action if we moved the pointer if next_correction_pos - last_correct_pos > 1: mouse_actions_sentence += 1 last_correct_pos = next_correction_pos # 2.3 Final check: The reference is a subset of the hypothesis: Cut the hypothesis if len(reference) < len(hypothesis): hypothesis = hypothesis[:len(reference)] errors_sentence += 1 logger.debug("Cutting hypothesis") # 2.4 Security assertion assert hypothesis == reference, "Error: The final hypothesis does not match with the reference! \n" \ "\t Split: %s \n" \ "\t Sentence: %d \n" \ "\t Hypothesis: %s\n" \ "\t Reference: %s" % (str(s.encode('utf-8')), n_line, hypothesis.encode('utf-8'), reference.encode('utf-8')) # 3. Update user effort counters mouse_actions_sentence += 1 # This +1 is the validation action chars_sentence = len(hypothesis) total_errors += errors_sentence total_mouse_actions += mouse_actions_sentence # 3.1 Log some info logger.debug(u"Final hypotesis: %s" % hypothesis) logger.debug( "%d errors. " "Sentence WSR: %4f. " "Sentence mouse strokes: %d " "Sentence MAR: %4f. " "Sentence MAR_c: %4f. " "Sentence KSMR: %4f. " % (errors_sentence, float(errors_sentence) / len(hypothesis), mouse_actions_sentence, float(mouse_actions_sentence) / len(hypothesis), float(mouse_actions_sentence) / chars_sentence, float(errors_sentence + mouse_actions_sentence) / chars_sentence)) # 5 Write correct sentences into a file return hypothesis, total_errors, total_mouse_actions
def on_batch_end(self, n_update, logs={}): self.cum_update += 1 if self.epoch_count + self.reload_epoch < self.start_sampling_on_epoch: return elif self.cum_update % self.each_n_updates != 0: return # Evaluate on each set separately for s in self.set_name: # Apply model predictions params_prediction = { 'batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars['n_parallel_loaders'], 'predict_on_sets': [s], 'n_samples': self.n_samples, 'pos_unk': False, 'heuristic': 0, 'mapping': None } if self.beam_search: params_prediction.update( checkDefaultParamsBeamSearch(self.extra_vars)) predictions, truths, sources = self.model_to_eval.predictBeamSearchNet( self.ds, params_prediction) else: # Convert predictions postprocess_fun = None if self.is_3DLabel: postprocess_fun = [ self.ds.convert_3DLabels_to_bboxes, self.extra_vars[s]['references_orig_sizes'] ] predictions = self.model_to_eval.predictNet( self.ds, params_prediction, postprocess_fun=postprocess_fun) if self.print_sources: if self.in_pred_idx is not None: sources = [srcs for srcs in sources[0][self.in_pred_idx]] sources = decode_predictions_beam_search(sources, self.index2word_x, pad_sequences=True, verbose=self.verbose) if s in predictions: if params_prediction['pos_unk']: samples = predictions[s][0] alphas = predictions[s][1] heuristic = params_prediction['heuristic'] else: samples = predictions[s] alphas = None heuristic = None predictions = predictions[s] if self.is_text: if self.out_pred_idx is not None: samples = samples[self.out_pred_idx] # Convert predictions into sentences if self.beam_search: predictions = decode_predictions_beam_search( samples, self.index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=params_prediction['mapping'], verbose=self.verbose) else: predictions = decode_predictions(samples, 1, self.index2word_y, self.sampling_type, verbose=self.verbose) truths = decode_predictions_one_hot(truths, self.index2word_y, verbose=self.verbose) # Write samples if self.print_sources: # Write samples for i, (source, sample, truth) in enumerate( zip(sources, predictions, truths)): print("Source (%d): %s" % (i, source)) print("Hypothesis (%d): %s" % (i, sample)) print("Reference (%d): %s" % (i, truth)) print("") else: for i, (sample, truth) in enumerate(zip(predictions, truths)): print("Hypothesis (%d): %s" % (i, sample)) print("Reference (%d): %s" % (i, truth)) print("")
def sample_ensemble(args, params): """ Use several translation models for obtaining predictions from a source text file. :param argparse.Namespace args: Arguments given to the method: * dataset: Dataset instance with data. * text: Text file with source sentences. * splits: Splits to sample. Should be already included in the dataset object. * dest: Output file to save scores. * weights: Weight given to each model in the ensemble. You should provide the same number of weights than models. By default, it applies the same weight to each model (1/N). * n_best: Write n-best list (n = beam size). * config: Config .pkl for loading the model configuration. If not specified, hyperparameters are read from config.py. * models: Path to the models. * verbose: Be verbose or not. :param params: parameters of the translation model. """ from data_engine.prepare_data import update_dataset_from_file from keras_wrapper.model_ensemble import BeamSearchEnsemble from keras_wrapper.cnn_model import loadModel from keras_wrapper.dataset import loadDataset from keras_wrapper.utils import decode_predictions_beam_search logger.info("Using an ensemble of %d models" % len(args.models)) models = [loadModel(m, -1, full_path=True) for m in args.models] dataset = loadDataset(args.dataset) dataset = update_dataset_from_file(dataset, args.text, params, splits=args.splits, remove_outputs=True) params['INPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len[params['INPUTS_IDS_DATASET'][0]] params['OUTPUT_VOCABULARY_SIZE'] = dataset.vocabulary_len[params['OUTPUTS_IDS_DATASET'][0]] # For converting predictions into sentences index2word_y = dataset.vocabulary[params['OUTPUTS_IDS_DATASET'][0]]['idx2words'] if params.get('APPLY_DETOKENIZATION', False): detokenize_function = eval('dataset.' + params['DETOKENIZATION_METHOD']) params_prediction = dict() params_prediction['max_batch_size'] = params.get('BATCH_SIZE', 20) params_prediction['n_parallel_loaders'] = params.get('PARALLEL_LOADERS', 1) params_prediction['beam_size'] = params.get('BEAM_SIZE', 6) params_prediction['maxlen'] = params.get('MAX_OUTPUT_TEXT_LEN_TEST', 100) params_prediction['optimized_search'] = params['OPTIMIZED_SEARCH'] params_prediction['model_inputs'] = params['INPUTS_IDS_MODEL'] params_prediction['model_outputs'] = params['OUTPUTS_IDS_MODEL'] params_prediction['dataset_inputs'] = params['INPUTS_IDS_DATASET'] params_prediction['dataset_outputs'] = params['OUTPUTS_IDS_DATASET'] params_prediction['search_pruning'] = params.get('SEARCH_PRUNING', False) params_prediction['normalize_probs'] = params.get('NORMALIZE_SAMPLING', False) params_prediction['alpha_factor'] = params.get('ALPHA_FACTOR', 1.0) params_prediction['coverage_penalty'] = params.get('COVERAGE_PENALTY', False) params_prediction['length_penalty'] = params.get('LENGTH_PENALTY', False) params_prediction['length_norm_factor'] = params.get('LENGTH_NORM_FACTOR', 0.0) params_prediction['coverage_norm_factor'] = params.get('COVERAGE_NORM_FACTOR', 0.0) params_prediction['pos_unk'] = params.get('POS_UNK', False) params_prediction['state_below_maxlen'] = -1 if params.get('PAD_ON_BATCH', True) \ else params.get('MAX_OUTPUT_TEXT_LEN', 50) params_prediction['output_max_length_depending_on_x'] = params.get('MAXLEN_GIVEN_X', True) params_prediction['output_max_length_depending_on_x_factor'] = params.get('MAXLEN_GIVEN_X_FACTOR', 3) params_prediction['output_min_length_depending_on_x'] = params.get('MINLEN_GIVEN_X', True) params_prediction['output_min_length_depending_on_x_factor'] = params.get('MINLEN_GIVEN_X_FACTOR', 2) params_prediction['attend_on_output'] = params.get('ATTEND_ON_OUTPUT', 'transformer' in params['MODEL_TYPE'].lower()) params_prediction['glossary'] = params.get('GLOSSARY', None) heuristic = params.get('HEURISTIC', 0) mapping = None if dataset.mapping == dict() else dataset.mapping model_weights = args.weights if args.glossary is not None: glossary = pkl2dict(args.glossary) elif params_prediction['glossary'] is not None: glossary = pkl2dict(params_prediction['glossary']) else: glossary = None if model_weights: assert len(model_weights) == len( models), 'You should give a weight to each model. You gave %d models and %d weights.' % ( len(models), len(model_weights)) model_weights = list(map(float, model_weights)) if len(model_weights) > 1: logger.info('Giving the following weights to each model: %s' % str(model_weights)) for s in args.splits: # Apply model predictions params_prediction['predict_on_sets'] = [s] beam_searcher = BeamSearchEnsemble(models, dataset, params_prediction, model_weights=model_weights, n_best=args.n_best, verbose=args.verbose) predictions = beam_searcher.predictBeamSearchNet()[s] samples = predictions['samples'] alphas = predictions['alphas'] if params_prediction['pos_unk'] else None if params_prediction['pos_unk']: sources = [x.strip() for x in open(args.text, 'r').read().split('\n')] sources = sources[:-1] if len(sources[-1]) == 0 else sources else: sources = None decoded_predictions = decode_predictions_beam_search(samples, index2word_y, glossary=glossary, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=mapping, verbose=args.verbose) # Apply detokenization function if needed if params.get('APPLY_DETOKENIZATION', False): decoded_predictions = list(map(detokenize_function, decoded_predictions)) if args.n_best: n_best_predictions = [] for i, (n_best_preds, n_best_scores, n_best_alphas) in enumerate(predictions['n_best']): n_best_sample_score = [] for n_best_pred, n_best_score, n_best_alpha in zip(n_best_preds, n_best_scores, n_best_alphas): pred = decode_predictions_beam_search([n_best_pred], index2word_y, glossary=glossary, alphas=[n_best_alpha] if params_prediction[ 'pos_unk'] else None, x_text=[sources[i]] if params_prediction['pos_unk'] else None, heuristic=heuristic, mapping=mapping, verbose=args.verbose) # Apply detokenization function if needed if params.get('APPLY_DETOKENIZATION', False): pred = list(map(detokenize_function, pred)) n_best_sample_score.append([i, pred, n_best_score]) n_best_predictions.append(n_best_sample_score) # Store result if args.dest is not None: filepath = args.dest # results file if params.get('SAMPLING_SAVE_MODE', 'list'): list2file(filepath, decoded_predictions) if args.n_best: nbest2file(filepath + '.nbest', n_best_predictions) else: raise Exception('Only "list" is allowed in "SAMPLING_SAVE_MODE"') else: list2stdout(decoded_predictions) if args.n_best: logger.info('Storing n-best sentences in ./' + s + '.nbest') nbest2file('./' + s + '.nbest', n_best_predictions) logger.info('Sampling finished')
def generate_sample(self, source_sentence, validated_prefix=None, max_N=5, isle_indices=None, filtered_idx2word=None, unk_indices=None, unk_words=None): """ Generate sample via constrained search. Options labeled with <<isles>> are untested and likely require some modifications to correctly work. :param source_sentence: Source sentence. :param validated_prefix: Prefix to keep in the output. :param max_N: Maximum number of words to generate between validated segments. <<isles>> :param isle_indices: Indices of the validated segments. <<isles>> :param filtered_idx2word: List of candidate words to be the next one to generate (after generating fixed_words). :param unk_indices: Positions of the unknown words. :param unk_words: Unknown words. :return: """ logger.log(2, 'Beam size: %d' % (self.params_prediction['beam_size'])) generate_sample_start_time = time.time() if unk_indices is None: unk_indices = [] if unk_words is None: unk_words = [] tokenization_start_time = time.time() tokenized_input = self.general_tokenize_f(source_sentence, escape=False) tokenized_input = self.model_tokenize_f(tokenized_input) tokenization_end_time = time.time() logger.log( 2, 'tokenization time: %.6f' % (tokenization_end_time - tokenization_start_time)) parse_input_start_time = time.time() # Go from text to indices src_seq = self.dataset.loadText( [tokenized_input], vocabularies=self.dataset.vocabulary[ self.params['INPUTS_IDS_DATASET'][0]], max_len=self.params['MAX_INPUT_TEXT_LEN'], offset=0, fill=self.dataset.fill_text[self.params['INPUTS_IDS_DATASET'][0]], pad_on_batch=self.dataset.pad_on_batch[ self.params['INPUTS_IDS_DATASET'][0]], words_so_far=False, loading_X=True)[0][0] parse_input_end_time = time.time() logger.log( 2, 'parse_input time: %.6f' % (parse_input_end_time - parse_input_start_time)) fixed_words_user = OrderedDict() unk_words_dict = OrderedDict() # If the user provided some feedback... if validated_prefix is not None: next_correction = validated_prefix[-1] if next_correction == self.eos_symbol: return validated_prefix[:-1].decode('utf-8') # 2.2.4 Tokenize the prefix properly (possibly applying BPE) # TODO: Here we are tokenizing the target language with the source language tokenizer prefix_tokenization_start_time = time.time() tokenized_validated_prefix = self.general_tokenize_f( validated_prefix, escape=False) tokenized_validated_prefix = self.model_tokenize_f( tokenized_validated_prefix) prefix_tokenization_end_time = time.time() logger.log( 2, 'prefix_tokenization time: %.6f' % (prefix_tokenization_end_time - prefix_tokenization_start_time)) # 2.2.5 Validate words word_validation_start_time = time.time() for pos, word in enumerate(tokenized_validated_prefix.split()): fixed_words_user[pos] = self.word2index_y.get( word, self.unk_id) if self.word2index_y.get(word) is None: unk_words_dict[pos] = word word_validation_end_time = time.time() logger.log( 2, 'word_validation time: %.6f' % (word_validation_end_time - word_validation_start_time)) # 2.2.6 Constrain search for the last word constrain_search_start_time = time.time() last_user_word_pos = list(fixed_words_user.keys())[-1] if next_correction != u' ': last_user_word = tokenized_validated_prefix.split()[-1] filtered_idx2word = dict( (self.word2index_y[candidate_word], candidate_word) for candidate_word in self.word2index_y if candidate_word[:len(last_user_word)] == last_user_word) if filtered_idx2word != dict(): del fixed_words_user[last_user_word_pos] if last_user_word_pos in list(unk_words_dict.keys()): del unk_words_dict[last_user_word_pos] else: filtered_idx2word = dict() constrain_search_end_time = time.time() logger.log( 2, 'constrain_search_end_time time: %.6f' % (constrain_search_end_time - constrain_search_start_time)) sample_beam_search_start_time = time.time() trans_indices, costs, alphas = \ self.interactive_beam_searcher.sample_beam_search_interactive(src_seq, fixed_words=copy.copy(fixed_words_user), max_N=max_N, isles=isle_indices, valid_next_words=filtered_idx2word, idx2word=self.index2word_y) sample_beam_search_end_time = time.time() logger.log( 2, 'sample_beam_search time: %.6f' % (sample_beam_search_end_time - sample_beam_search_start_time)) if False and self.params_prediction['pos_unk']: alphas = [alphas] sources = [tokenized_input] heuristic = self.params_prediction['heuristic'] else: alphas = None heuristic = None sources = None # 1.2 Decode hypothesis decoding_predictions_start_time = time.time() hypothesis = decode_predictions_beam_search([trans_indices], self.index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=self.mapping, pad_sequences=True, verbose=0)[0] decoding_predictions_end_time = time.time() logger.log( 2, 'decoding_predictions time: %.6f' % (decoding_predictions_end_time - decoding_predictions_start_time)) # UNK words management unk_management_start_time = time.time() unk_indices = list(unk_words_dict) unk_words = list(unk_words_dict.values()) if len(unk_indices) > 0: # If we added some UNK word hypothesis = hypothesis.split() if len(hypothesis) < len( unk_indices ): # The full hypothesis will be made up UNK words: for i, index in enumerate(range(0, len(hypothesis))): hypothesis[index] = unk_words[unk_indices[i]] for ii in range(i + 1, len(unk_words)): hypothesis.append(unk_words[ii]) else: # We put each unknown word in the corresponding gap for i, index in enumerate(unk_indices): if index < len(hypothesis): hypothesis[index] = unk_words[i] else: hypothesis.append(unk_words[i]) hypothesis = u' '.join(hypothesis) unk_management_end_time = time.time() logger.log( 2, 'unk_management time: %.6f' % (unk_management_end_time - unk_management_start_time)) hypothesis_detokenization_start_time = time.time() hypothesis = self.model_detokenize_f(hypothesis) hypothesis = self.general_detokenize_f(hypothesis, unescape=False) hypothesis_detokenization_end_time = time.time() logger.log( 2, 'hypothesis_detokenization time: %.6f' % (hypothesis_detokenization_end_time - hypothesis_detokenization_start_time)) generate_sample_end_time = time.time() logger.log( 2, 'generate_sample time: %.6f' % (generate_sample_end_time - generate_sample_start_time)) return hypothesis
def evaluate(self, epoch, counter_name='epoch', logs=None): if logs is None: logs = {} # Change inputs and outputs mappings for evaluation self.changeInOutMappings() # Evaluate on each set separately all_metrics = [] for s in self.set_name: # Apply model predictions if self.beam_search: params_prediction = {'max_batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars[ 'n_parallel_loaders'], 'predict_on_sets': [s], 'beam_batch_size': self.beam_batch_size if self.beam_batch_size is not None else self.batch_size, 'pos_unk': False, 'normalize': self.normalize, 'normalization_type': self.normalization_type, 'max_eval_samples': self.max_eval_samples } params_prediction.update(checkDefaultParamsBeamSearch(self.extra_vars)) predictions_all = self.model_to_eval.predictBeamSearchNet(self.ds, params_prediction)[s] else: orig_size = self.extra_vars.get('eval_orig_size', False) params_prediction = {'batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars.get( 'n_parallel_loaders', 8), 'predict_on_sets': [s], 'normalize': self.normalize, 'normalization_type': self.normalization_type, 'max_eval_samples': self.max_eval_samples, 'model_name': self.model_name, } # Convert predictions postprocess_fun = None if self.is_3DLabel: postprocess_fun = [self.ds.convert_3DLabels_to_bboxes, self.extra_vars[s]['references_orig_sizes']] elif orig_size: postprocess_fun = [self.ds.resize_semantic_output, self.extra_vars[s]['eval_orig_size_id']] predictions_all = \ self.model_to_eval.predictNet(self.ds, params_prediction, postprocess_fun=postprocess_fun)[s] # Single-output model if not self.gt_pos or self.gt_pos == 0 or len(self.gt_pos) == 1: if len(predictions_all) != 2: predictions_all = [predictions_all] gt_positions = [0] # Multi-output model else: gt_positions = self.gt_pos # Select each output to evaluate separately for gt_pos, type, these_metrics, gt_id, write_type, index2word_y, index2word_x in zip( gt_positions, self.output_types, self.metric_name, self.gt_id, self.write_type, self.index2word_y, self.index2word_x): predictions = predictions_all[gt_pos] if self.verbose > 0: print('') logging.info('Prediction output ' + str(gt_pos) + ': ' + str( gt_id) + ' (' + str(type) + ')') # Postprocess outputs of type text if type == 'text': if params_prediction.get('pos_unk', False): samples = predictions[0] alphas = predictions[1] if eval('self.ds.loaded_raw_' + s + '[0]'): sources = predictions[2] else: sources = [] for preds in predictions[2]: for src in preds[self.input_text_id]: sources.append(src) sources = decode_predictions_beam_search(sources, index2word_x, pad_sequences=True, verbose=self.verbose) heuristic = self.extra_vars['heuristic'] else: samples = predictions alphas = None heuristic = None sources = None if self.out_pred_idx is not None: samples = samples[self.out_pred_idx] # Convert predictions into sentences if self.beam_search: predictions = decode_predictions_beam_search(samples, index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=self.extra_vars.get( 'mapping', None), verbose=self.verbose) else: probs = predictions predictions = decode_predictions(predictions, 1, # always set temperature to 1 index2word_y, self.sampling_type, verbose=self.verbose) # Apply detokenization function if needed if self.extra_vars.get('apply_detokenization', False): predictions = map(self.extra_vars['detokenize_f'], predictions) # Postprocess outputs of type binary elif type == 'binary': predictions = decode_multilabel(predictions, index2word_y, min_val=self.min_pred_multilabel[ gt_pos], verbose=self.verbose) # Prepare references # exec ("y_raw = self.ds.Y_" + s + "[gt_id]") y_split = getattr(self.ds, 'Y_' + s) y_raw = y_split[gt_id] self.extra_vars[gt_pos][s]['references'] = self.ds.loadBinary(y_raw, gt_id) # Postprocess outputs of type 3DLabel elif type == '3DLabel': self.extra_vars[gt_pos][s] = dict() # exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]') y_split = getattr(self.ds, 'Y_' + s) ref = y_split[gt_id] [ref, original_sizes] = self.ds.convert_GT_3DLabels_to_bboxes( ref) self.extra_vars[gt_pos][s]['references'] = ref self.extra_vars[gt_pos][s]['references_orig_sizes'] = original_sizes # Postprocess outputs of type 3DSemanticLabel elif type == '3DSemanticLabel': self.extra_vars[gt_pos]['eval_orig_size'] = self.eval_orig_size self.extra_vars[gt_pos][s] = dict() # exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]') y_split = getattr(self.ds, 'Y_' + s) ref = y_split[gt_id] if self.eval_orig_size: old_crop = copy.deepcopy(self.ds.img_size_crop) self.ds.img_size_crop = copy.deepcopy(self.ds.img_size) self.extra_vars[gt_pos][s]['eval_orig_size_id'] = np.array([gt_id] * len(ref)) ref = self.ds.load_GT_3DSemanticLabels(ref, gt_id) if self.eval_orig_size: self.ds.img_size_crop = copy.deepcopy(old_crop) self.extra_vars[gt_pos][s]['references'] = ref # Other output data types else: # exec ("self.extra_vars[gt_pos][s]['references'] = self.ds.Y_" + s + "[gt_id]") y_split = getattr(self.ds, 'Y_' + s) self.extra_vars[gt_pos][s]['references'] = y_split[gt_id] # Store predictions if self.write_samples: # Store result filepath = self.save_path + '/' + s + '_' + counter_name + '_' + str(epoch) + '_output_' + str(gt_pos) + '.pred' # results file if write_type == 'list': list2file(filepath, predictions) elif write_type == 'vqa': try: # exec ('refs = self.ds.Y_' + s + '[gt_id]') y_split = getattr(self.ds, 'Y_' + s) refs = y_split[gt_id] except Exception: refs = ['N/A' for _ in range(probs.shape[0])] extra_data_plot = {'reference': refs, 'probs': probs, 'vocab': index2word_y} list2vqa(filepath, predictions, self.extra_vars[gt_pos][s]['question_ids'], extra=extra_data_plot) elif write_type == 'listoflists': listoflists2file(filepath, predictions) elif write_type == 'numpy': numpy2file(filepath, predictions) elif write_type == '3DLabels': raise NotImplementedError( 'Write 3DLabels function is not implemented') elif write_type == '3DSemanticLabel': folder_path = self.save_path + '/' + s + '_' + counter_name + '_' + str( epoch) # results folder numpy2imgs(folder_path, predictions, eval('self.ds.X_' + s + '["' + self.input_id + '"]'), self.ds) else: raise NotImplementedError('The store type "' + self.write_type + '" is not implemented.') # Evaluate on each metric for metric in these_metrics: if self.verbose > 0: logging.info('Evaluating on metric ' + metric) filepath = self.save_path + '/' + s + '.' + metric # results file if s == 'train': logging.info( "WARNING: evaluation results on 'train' split might be incorrect when" "applying random image shuffling.") # Evaluate on the chosen metric metrics = evaluation.select[metric]( pred_list=predictions, verbose=self.verbose, extra_vars=self.extra_vars[gt_pos], split=s) # Print results to file and store in model log with open(filepath, 'a') as f: header = counter_name + ',' line = str(epoch) + ',' # Store in model log self.model_to_eval.log(s, counter_name, epoch) for metric_ in sorted(metrics): value = metrics[metric_] # Multiple-output model if self.gt_pos and self.gt_pos != 0: metric_ += '_output_' + str(gt_pos) all_metrics.append(metric_) header += metric_ + ',' line += str(value) + ',' # Store in model log self.model_to_eval.log(s, metric_, value) if not self.written_header: f.write(header + '\n') self.written_header = True f.write(line + '\n') if self.verbose > 0: logging.info('Done evaluating on metric ' + metric) # Store losses if logs.get('loss') is not None: self.model_to_eval.log('train', 'train_loss', logs['loss']) if logs.get('valid_loss') is not None: self.model_to_eval.log('val', 'val_loss', logs['valid_loss']) # Plot results so far if self.do_plot: if self.metric_name: self.model_to_eval.plot(counter_name, set(all_metrics), self.set_name, upperbound=self.max_plot) # Save the model if self.save_each_evaluation: from keras_wrapper.cnn_model import saveModel saveModel(self.model_to_eval, epoch, store_iter=not self.eval_on_epochs) # Recover inputs and outputs mappings for resume training self.recoverInOutMappings()