def rescore_model(source_file, target_file, saveto, models, options, b, normalization_alpha, verbose, alignweights): trng = RandomStreams(1234) def _score(pairs, alignweights=False): # sample given an input sequence and obtain scores scores = [] alignments = [] for i, model in enumerate(models): f_log_probs = load_scorer(model, options[i], alignweights=alignweights) score, alignment = pred_probs(f_log_probs, prepare_data, options[i], pairs, normalization_alpha=normalization_alpha, alignweights = alignweights) scores.append(score) alignments.append(alignment) return scores, alignments pairs = TextIterator(source_file.name, target_file.name, options[0]['dictionaries'][:-1], options[0]['dictionaries'][-1], n_words_source=options[0]['n_words_src'], n_words_target=options[0]['n_words'], batch_size=b, maxlen=float('inf'), sort_by_length=False) #TODO: sorting by length could be more efficient, but we'd want to resort after scores, alignments = _score(pairs, alignweights) source_file.seek(0) target_file.seek(0) source_lines = source_file.readlines() target_lines = target_file.readlines() for i, line in enumerate(target_lines): score_str = ' '.join(map(str,[s[i] for s in scores])) if verbose: saveto.write('{0} '.format(line.strip())) saveto.write('{0}\n'.format(score_str)) ### optional save weights mode. if alignweights: ### writing out the alignments. temp_name = saveto.name + ".json" with tempfile.NamedTemporaryFile(prefix=temp_name) as align_OUT: for line in all_alignments: align_OUT.write(line + "\n") ### combining the actual source and target words. combine_source_target_text_1to1(source_file, target_file, saveto.name, align_OUT)
def multi_rescore_model(source_file, target_file, savetos, models, options, b, normalization_alpha, verbose, alignweights, extra_sources=[], per_word=False): trng = RandomStreams(1234) def _score(pairs, alignweights=False): # sample given an input sequence and obtain scores scores = [] #alignments = [] #aux_alignments = [] costs_per_word = [] for i, model in enumerate(models): f_log_probs = load_scorer(model, options[i], alignweights=alignweights) score, all_alignments, cost_per_word = multi_pred_probs( f_log_probs, prepare_multi_data, options[i], pairs, normalization_alpha=normalization_alpha, alignweights=alignweights) #print 'alignment lens' #print len(all_alignments) #print len(all_alignments[0]) scores.append(score) costs_per_word.append(cost_per_word) return scores, tuple(all_alignments), costs_per_word #print 'extra_sources', extra_sources # list of sources + target sentences (target sentences are the final list) # TODO: make TextIterator generic sents = TextIterator(source_file.name, target_file.name, options[0]['dictionaries'][:-1], options[0]['dictionaries'][-1], n_words_source=options[0]['n_words_src'], n_words_target=options[0]['n_words'], batch_size=b, maxlen=float('inf'), sort_by_length=False, extra_sources=[ss.name for ss in extra_sources]) # TODO: sorting by length could be more efficient, but we'd want to resort after scores, all_alignments, costs_per_word = _score(sents, alignweights) source_lines = [] source_file.seek(0) source_lines.append([source_file.readlines()]) extra_source_lines = [] for i, ss in enumerate(extra_sources): extra_sources[i].seek(0) extra_source_lines.append([extra_sources[i].readlines()]) target_file.seek(0) target_lines = target_file.readlines() # print out scores for each translation for i, line in enumerate(target_lines): if per_word: score_str = ' '.join( map(str, [s for s in costs_per_word[0][i] ][:len(line.split(" ")) + 1])) else: score_str = ' '.join(map(str, [s[i] for s in scores])) if verbose: savetos[0].write('{0} '.format(line.strip())) savetos[0].write('{0}\n'.format(score_str)) # optional save weights mode. if alignweights: #print 'num alignments', len(all_alignments) for i, alignments in enumerate(all_alignments): # write out the alignments. #print len(alignments) temp_name = savetos[i].name + str(i) + ".json" #print temp_name with tempfile.NamedTemporaryFile(prefix=temp_name) as align_OUT: for line in alignments: #print len(line[0][0]) #raw_input() align_OUT.write(line + "\n") # combine the actual source and target words. #print 'savetos', len(savetos) #print 'source files', len(extra_sources) if i == 0: tmp_srcfile = source_file else: tmp_srcfile = extra_sources[i - 1] combine_source_target_text_1to1(tmp_srcfile, target_file, savetos[i].name, align_OUT, suffix=str(i))
def rescore_model(source_file, target_file, saveto, models, options, b, normalization_alpha, verbose, alignweights): trng = RandomStreams(1234) fs_log_probs = [] for model, option in zip(models, options): # load model parameters and set theano shared variables param_list = numpy.load(model).files param_list = dict.fromkeys( [key for key in param_list if not key.startswith('adam_')], 0) params = load_params(model, param_list) tparams = init_theano_params(params) trng, use_noise, \ x, x_mask, y, y_mask, \ opt_ret, \ cost = \ build_model(tparams, option) inps = [x, x_mask, y, y_mask] use_noise.set_value(0.) if alignweights: logging.debug( "Save weight mode ON, alignment matrix will be saved.") outputs = [cost, opt_ret['dec_alphas']] f_log_probs = theano.function(inps, outputs) else: f_log_probs = theano.function(inps, cost) fs_log_probs.append(f_log_probs) def _score(pairs, alignweights=False): # sample given an input sequence and obtain scores scores = [] alignments = [] for i, f_log_probs in enumerate(fs_log_probs): score, alignment = pred_probs( f_log_probs, prepare_data, options[i], pairs, normalization_alpha=normalization_alpha, alignweights=alignweights) scores.append(score) alignments.append(alignment) return scores, alignments pairs = TextIterator( source_file.name, target_file.name, options[0]['dictionaries'][:-1], options[0]['dictionaries'][-1], n_words_source=options[0]['n_words_src'], n_words_target=options[0]['n_words'], batch_size=b, maxlen=float('inf'), sort_by_length=False ) #TODO: sorting by length could be more efficient, but we'd want to resort after scores, alignments = _score(pairs, alignweights) source_file.seek(0) target_file.seek(0) source_lines = source_file.readlines() target_lines = target_file.readlines() for i, line in enumerate(target_lines): score_str = ' '.join(map(str, [s[i] for s in scores])) if verbose: saveto.write('{0} '.format(line.strip())) saveto.write('{0}\n'.format(score_str)) ### optional save weights mode. if alignweights: ### writing out the alignments. temp_name = saveto.name + ".json" with tempfile.NamedTemporaryFile(prefix=temp_name) as align_OUT: for line in all_alignments: align_OUT.write(line + "\n") ### combining the actual source and target words. combine_source_target_text_1to1(source_file, target_file, saveto.name, align_OUT)