def prepare_data_seq(batch_size=32):
pairs_tra, pairs_val, pairs_tst, vocab = load_dataset()
logging.info("Vocab {} ".format(vocab.n_words))
dataset_train = Dataset(pairs_tra, vocab)
data_loader_tra = torch.utils.data.DataLoader(dataset=dataset_train,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_fn,
drop_last=True)
dataset_valid = Dataset(pairs_val, vocab)
data_loader_val = torch.utils.data.DataLoader(dataset=dataset_valid,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_fn,
drop_last=True)
#print('val len:',len(dataset_valid))
dataset_test = Dataset(pairs_tst, vocab)
data_loader_tst = torch.utils.data.DataLoader(dataset=dataset_test,
batch_size=1,
shuffle=False,
collate_fn=collate_fn,
drop_last=True)
write_config()
return data_loader_tra, data_loader_val, data_loader_tst, vocab, len(
dataset_train.emo_map)
def prepare_data_seq(batch_size=32):
pairs_tra, pairs_val, pairs_tst, vocab = load_dataset()
logging.info("Vocab {} ".format(vocab.n_words))
dataset_train = Dataset(pairs_tra, vocab)
data_loader_tra = torch.utils.data.DataLoader(dataset=dataset_train,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_fn)
dataset_valid = Dataset(pairs_val, vocab)
data_loader_val = torch.utils.data.DataLoader(dataset=dataset_valid,
batch_size=batch_size,
shuffle=True,
collate_fn=collate_fn)
#print('val len:',len(dataset_valid))
'''
print("Dataset for tst i.e., the keys of the dict : ",pairs_tst.keys())
print("Dataset for tst i.e., context values of pairs_tst : ",pairs_tst['context'][0:2])
print("Dataset for tst i.e., target values of pairs_tst : ",pairs_tst['target'][0:2])
print("Dataset for tst i.e., emotion values of pairs_tst : ",pairs_tst['emotion'][0:2])
print("Dataset for tst i.e., situation values of pairs_tst : ",pairs_tst['situation'][0:2])
print("==================================================================================")
print("Dataset for tst i.e., context values of pairs_tst : ",pairs_tst['context'][2:4])
print("Dataset for tst i.e., target values of pairs_tst : ",pairs_tst['target'][2:4])
print("Dataset for tst i.e., emotion values of pairs_tst : ",pairs_tst['emotion'][2:4])
print("Dataset for tst i.e., situation values of pairs_tst : ",pairs_tst['situation'][2:4])
'''
dataset_test = Dataset(pairs_tst, vocab)
data_loader_tst = torch.utils.data.DataLoader(dataset=dataset_test,
batch_size=1,
shuffle=False,
collate_fn=collate_fn)
write_config()
print(type(vocab))
return data_loader_tra, data_loader_val, data_loader_tst, vocab, len(
dataset_train.emo_map)
def prepare_data_seq(batch_size=config.batch_size):
pairs_tra, pairs_val, vocab = load_dataset()
print("Number of train data",len(pairs_tra['target']))
logging.info("Vocab {} ".format(vocab.n_words))
dataset_train = Dataset(pairs_tra, vocab)
data_loader_tra = torch.utils.data.DataLoader(dataset=dataset_train,
batch_size=batch_size,
shuffle=True, collate_fn=collate_fn)
dataset_valid = Dataset(pairs_val, vocab)
data_loader_val = torch.utils.data.DataLoader(dataset=dataset_valid,
batch_size=batch_size,
shuffle=False, collate_fn=collate_fn)
# #print('val len:',len(dataset_valid))
# dataset_test = Dataset(pairs_tst, vocab)
# data_loader_tst = torch.utils.data.DataLoader(dataset=dataset_test,
# batch_size=1,
# shuffle=False, collate_fn=collate_fn)
write_config()
#return data_loader_tra, data_loader_val, data_loader_tst, vocab, len(dataset_train.emo_map)
return data_loader_tra, data_loader_val, vocab, len(dataset_train.emo_map)