num_iterations,
src_corpus_tags=src_corpus_tags,
trg_corpus_tags=trg_corpus_tags)
return align_corpus_given_models(src_corpus,
trg_corpus,
prior_model,
translation_model,
src_corpus_tags=src_corpus_tags,
trg_corpus_tags=trg_corpus_tags)
if __name__ == "__main__":
if not len(sys.argv) == 6:
print "Usage ./word_alignment.py src_corpus trg_corpus iterations output_prefix."
sys.exit(0)
src_corpus = utils.read_all_tokens(sys.argv[1])
trg_corpus = utils.read_all_tokens(sys.argv[2])
with_tags = sys.argv[5] == 'tags'
num_iterations = int(sys.argv[3])
output_prefix = sys.argv[4]
assert len(src_corpus) == len(trg_corpus), "Corpora should be same size!"
if with_tags:
src_corpus_tags = utils.read_all_tokens(
sys.argv[1][:sys.argv[1].find('tokens')] + 'tags.' +
sys.argv[1][sys.argv[1].find('1'):])
if sys.argv[2].find('tokens') == -1:
trg_corpus_tags = utils.read_all_tokens(
sys.argv[2][:sys.argv[2].find('lemmas')] + 'tags.' +
sys.argv[2][sys.argv[2].find('1'):])
else:
def initialize_models(src_corpus, trg_corpus):
prior_model = PriorModel(src_corpus, trg_corpus)
translation_model = TranslationModel(src_corpus, trg_corpus)
return prior_model, translation_model
def normalize(src_corpus, trg_corpus):
return src_corpus, trg_corpus
if __name__ == "__main__":
if not len(sys.argv) == 5:
print(
"Usage ./align.py src_corpus trg_corpus iterations output_prefix.")
sys.exit(0)
src_corpus, trg_corpus = read_all_tokens(sys.argv[1]), read_all_tokens(
sys.argv[2])
src_corpus, trg_corpus = normalize(src_corpus, trg_corpus)
num_iterations = int(sys.argv[3])
output_prefix = sys.argv[4]
assert len(src_corpus) == len(trg_corpus), "Corpora should be same size!"
prior_model, translation_model = initialize_models(src_corpus, trg_corpus)
prior_model, translation_model = estimate_models(src_corpus, trg_corpus,
prior_model,
translation_model,
num_iterations)
alignments = align_corpus(src_corpus, trg_corpus, prior_model,
translation_model)
output_alignments_per_test_set(alignments, output_prefix)
these_alignments = align_sentence_pair(src_corpus[i], trg_corpus[i],
prior_model, translation_model)
alignments.append(these_alignments)
return alignments
def align_corpus(src_corpus, trg_corpus, num_iterations):
"Learn models and then align the corpus using them."
prior_model, translation_model = initialize_models(src_corpus, trg_corpus)
prior_model, translation_model = estimate_models(src_corpus, trg_corpus,
prior_model,
translation_model,
num_iterations)
return align_corpus_given_models(src_corpus, trg_corpus, prior_model,
translation_model)
if __name__ == "__main__":
if not len(sys.argv) == 5:
print(
"Usage: python word_alignment.py src_corpus trg_corpus iterations output_prefix."
)
sys.exit(0)
src_corpus = utils.read_all_tokens(sys.argv[1])
trg_corpus = utils.read_all_tokens(sys.argv[2])
num_iterations = int(sys.argv[3])
output_prefix = sys.argv[4]
assert len(src_corpus) == len(trg_corpus), "Corpora should be same size!"
alignments = align_corpus(src_corpus, trg_corpus, num_iterations)
utils.output_alignments_per_test_set(alignments, output_prefix)