"examples/wmt_2018/en_lv/data/en_lv",
"examples/wmt_2018/en_lv/data/en_lv", "smt"
],
}
train_list = []
dev_list = []
test_list = []
index_list = []
test_sentence_pairs_list = []
for key, value in languages.items():
if value[3] == "nmt":
train_temp = read_annotated_file(path=value[0],
original_file="train.nmt.src",
translation_file="train.nmt.mt",
hter_file="train.nmt.hter")
dev_temp = read_annotated_file(path=value[1],
original_file="dev.nmt.src",
translation_file="dev.nmt.mt",
hter_file="dev.nmt.hter")
test_temp = read_test_file(path=value[2],
original_file="test.nmt.src",
translation_file="test.nmt.mt")
elif value[3] == "smt":
train_temp = read_annotated_file(path=value[0],
original_file="train.smt.src",
translation_file="train.smt.mt",
hter_file="train.smt.hter")
dev_temp = read_annotated_file(path=value[1],
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO,
handlers=[LoggingHandler()])
if not os.path.exists(TEMP_DIRECTORY):
os.makedirs(TEMP_DIRECTORY)
if GOOGLE_DRIVE:
download_from_google_drive(DRIVE_FILE_ID, MODEL_NAME)
TRAIN_FOLDER = "examples/wmt_2018/en_de/data/en_de"
DEV_FOLDER = "examples/wmt_2018/en_de/data/en_de"
TEST_FOLDER = "examples/wmt_2018/en_de/data/en_de"
train = read_annotated_file(path=TRAIN_FOLDER,
original_file="train.smt.src",
translation_file="train.smt.mt",
hter_file="train.smt.hter")
dev = read_annotated_file(path=DEV_FOLDER,
original_file="dev.smt.src",
translation_file="dev.smt.mt",
hter_file="dev.smt.hter")
test = read_test_file(path=TEST_FOLDER,
original_file="test.smt.src",
translation_file="test.smt.mt")
index = test['index'].to_list()
train = train[['original', 'translation', 'hter']]
dev = dev[['original', 'translation', 'hter']]
test = test[['original', 'translation']]