index_list = []
test_sentence_pairs_list = []
for key, value in languages.items():
if value[3] == "nmt":
train_temp = read_annotated_file(path=value[0],
original_file="train.nmt.src",
translation_file="train.nmt.mt",
hter_file="train.nmt.hter")
dev_temp = read_annotated_file(path=value[1],
original_file="dev.nmt.src",
translation_file="dev.nmt.mt",
hter_file="dev.nmt.hter")
test_temp = read_test_file(path=value[2],
original_file="test.nmt.src",
translation_file="test.nmt.mt")
elif value[3] == "smt":
train_temp = read_annotated_file(path=value[0],
original_file="train.smt.src",
translation_file="train.smt.mt",
hter_file="train.smt.hter")
dev_temp = read_annotated_file(path=value[1],
original_file="dev.smt.src",
translation_file="dev.smt.mt",
hter_file="dev.smt.hter")
test_temp = read_test_file(path=value[2],
original_file="test.smt.src",
translation_file="test.smt.mt")
download_from_google_drive(DRIVE_FILE_ID, MODEL_NAME)
TRAIN_FOLDER = "examples/wmt_2018/en_de/data/en_de"
DEV_FOLDER = "examples/wmt_2018/en_de/data/en_de"
TEST_FOLDER = "examples/wmt_2018/en_de/data/en_de"
train = read_annotated_file(path=TRAIN_FOLDER,
original_file="train.smt.src",
translation_file="train.smt.mt",
hter_file="train.smt.hter")
dev = read_annotated_file(path=DEV_FOLDER,
original_file="dev.smt.src",
translation_file="dev.smt.mt",
hter_file="dev.smt.hter")
test = read_test_file(path=TEST_FOLDER,
original_file="test.smt.src",
translation_file="test.smt.mt")
index = test['index'].to_list()
train = train[['original', 'translation', 'hter']]
dev = dev[['original', 'translation', 'hter']]
test = test[['original', 'translation']]
train = train.rename(columns={
'original': 'text_a',
'translation': 'text_b',
'hter': 'labels'
}).dropna()
dev = dev.rename(columns={
'original': 'text_a',
