train_fname = 'bAbI_Data/'+str(train_foldername)+'/'+str(train_filename)
valid_foldername = 'en-valid-10k'
valid_filename = dataset + '_valid'
valid_fname = 'bAbI_Data/'+str(valid_foldername)+'/'+str(valid_filename)
test_foldername = 'en-valid-10k'
test_filename = dataset + '_test'
test_fname = 'bAbI_Data/'+str(test_foldername)+'/'+str(test_filename)
vec_fname = 'bAbI_Data/model.vec'
unk_thres = 0
pre_embed = False
train_data_BOW, valid_data_BOW, test_data_BOW, train_data_pe, valid_data_pe, test_data_pe, vocab = data_transform.get_data(
train_fname, valid_fname, test_fname, vec_fname=vec_fname, unk_thres = unk_thres, pre_embed=pre_embed)
# print(train_data_pe[0:5])
# In[25]:
databag = []
model = torch.load('saved_models/' + model_name)
def test_visualize(model,test_dt_bow,test_dt_pe):
this_story = []
test_shape = test_dt_bow.shape
n_corr = 0;
count = 0;
lim = 2000
with open('variables/word2idx','rb') as handle:
#Loading data
print("load...")
if data_type == "txt":
data = np.loadtxt(fname, dtype='float64')
elif data_type == "dat":
data = np.fromfile(fname, dtype='float64')
elif data_type == "csv":
data = np.genfromtxt(fname, delimiter=',', dtype='float64')
'''
Transform Data in data_transform.py
'''
data_transform.load_data(data)
data_transform.main()
data = data_transform.get_data()
'''
Parameter Part and Global variable
'''
#declare variables
training = parameter.forward_training
training['Data'] = data[int(np.floor(data.shape[0] * training['Region'])):int(
np.floor(data.shape[0] * training['Region']) +
np.floor(data.shape[0] * training['Rate']))]
training['BatchSize'] = int(np.floor(training['Data'].shape[0]))
inputLen = parameter.forward_inputLen
##input
def get_data(folder_name, qa_name, unk_thres=0, pre_embed=False):
train_foldername = folder_name
train_filename = qa_name + '_train'
train_fname = 'bAbI_Data/' + str(train_foldername) + '/' + str(
train_filename)
valid_foldername = folder_name
valid_filename = qa_name + '_valid'
valid_fname = 'bAbI_Data/' + str(valid_foldername) + '/' + str(
valid_filename)
test_foldername = folder_name
test_filename = qa_name + '_test'
test_fname = 'bAbI_Data/' + str(test_foldername) + '/' + str(test_filename)
vec_fname = 'bAbI_Data/model.vec'
unk_thres = 0
pre_embed = pre_embed
train_data_BOW, valid_data_BOW, test_data_BOW, train_data_pe, valid_data_pe, test_data_pe, vocab = data_transform.get_data(
train_fname,
valid_fname,
test_fname,
vec_fname=vec_fname,
unk_thres=unk_thres,
pre_embed=pre_embed)
print('Train data size : ', train_data_BOW.shape)
print('Valid data size : ', valid_data_BOW.shape)
print('Test data size : ', test_data_BOW.shape)
print('Vocab size : ', len(vocab))
return train_data_BOW, valid_data_BOW, test_data_BOW, train_data_pe, valid_data_pe, test_data_pe, vocab
