pairs = [sample[0] for sample in batch]
input_lengths = [sample[1][0] for sample in batch]
target_lengths = [sample[1][1] for sample in batch]
max_input_length = max(input_lengths)
max_target_length = max(target_lengths)
pairstensor = tensorsFromPairsSorted(lang_vi, lang_en_vi, max_input_length,
max_target_length, pairs)
return [pairstensor[0], \
(torch.from_numpy(np.array(pairstensor[1]))), \
pairstensor[2], \
(torch.from_numpy(np.array(pairstensor[3])))]
train_vi = langDataset(train_id_vi)
val_vi = langDataset(val_id_vi)
test_vi = langDataset(test_id_vi)
train_loader_vi = torch.utils.data.DataLoader(dataset=train_vi,
batch_size=batch_size,
collate_fn=collate_fn_vi,
shuffle=True)
# train_loader_vi = torch.utils.data.DataLoader(dataset=train_vi,
# batch_size=batch_size,
# collate_fn=collate_fn_vi,
# shuffle=True)
val_loader_vi = torch.utils.data.DataLoader(dataset=val_vi,
batch_size=batch_size,
collate_fn=collate_fn_vi,
shuffle=True)
pairs = [sample[0] for sample in batch]
input_lengths = [sample[1][0] for sample in batch]
target_lengths = [sample[1][1] for sample in batch]
max_input_length = max(input_lengths)
max_target_length = max(target_lengths)
pairstensor = tensorsFromPairsSorted(lang_zh, lang_en_zh, max_input_length,
max_target_length, pairs)
return [pairstensor[0], \
(torch.from_numpy(np.array(pairstensor[1]))), \
pairstensor[2], \
(torch.from_numpy(np.array(pairstensor[3])))]
train_zh = langDataset(train_id_zh)
val_zh = langDataset(val_id_zh)
test_zh = langDataset(test_id_zh)
train_loader_zh = torch.utils.data.DataLoader(dataset=train_zh,
batch_size=batch_size,
collate_fn=collate_fn_zh,
shuffle=True)
val_loader_zh = torch.utils.data.DataLoader(dataset=val_zh,
batch_size=batch_size,
collate_fn=collate_fn_zh,
shuffle=True)
test_loader_zh = torch.utils.data.DataLoader(dataset=test_zh,
batch_size=batch_size,
collate_fn=collate_fn_zh,
shuffle=True)