def get_model():
importlib.reload(d_data)
importlib.reload(IE_data)
d_metadata, d_idx_q, d_idx_a = d_data.load_data(PATH='../datasets/danny/')
i_metadata, i_idx_q, i_idx_a = IE_data.load_data(PATH='../datasets/IE/')
(d_trainX, d_trainY), (d_testX, d_testY), (d_validX, d_validY) = data_utils.split_dataset(d_idx_q, d_idx_a)
(i_trainX, i_trainY), (i_testX, i_testY), (i_validX, i_validY) = data_utils.split_dataset(i_idx_q, i_idx_a)
d_model = seq2seq_wrapper.Seq2Seq(
xseq_len=d_trainX.shape[-1],
yseq_len=d_trainY.shape[-1],
xvocab_size=len(d_metadata['idx2w']),
yvocab_size=len(d_metadata['idx2w']),
ckpt_path='../ckpt/danny/',
loss_path='',
metadata=d_metadata,
emb_dim=1024,
num_layers=3
)
i_model = seq2seq_wrapper.Seq2Seq(
xseq_len=i_trainX.shape[-1],
yseq_len=i_trainY.shape[-1],
xvocab_size=len(i_metadata['idx2w']),
yvocab_size=len(i_metadata['idx2w']),
ckpt_path='../ckpt/IE/',
loss_path='',
metadata=i_metadata,
emb_dim=1024,
num_layers=3
)
d_sess = d_model.restore_last_session()
i_sess = i_model.restore_last_session()
return d_model, i_model, d_sess, i_sess, d_metadata, i_metadata
# preprocessed data
from datasets.danny import data
import data_utils
# load data from pickle and npy files
metadata, idx_q, idx_a = data.load_data(PATH='datasets/danny/')
(trainX, trainY), (testX,
testY), (validX,
validY) = data_utils.split_dataset(idx_q, idx_a)
# parameters
xseq_len = trainX.shape[-1]
yseq_len = trainY.shape[-1]
batch_size = 32
xvocab_size = len(metadata['idx2w'])
yvocab_size = xvocab_size
emb_dim = 1024
import seq2seq_wrapper
# In[7]:
model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
yseq_len=yseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
ckpt_path='ckpt/danny/',
loss_path='ckpt/danny/preset/',
metadata=metadata,
emb_dim=emb_dim,
num_layers=3,
# preprocessed data
from datasets.danny import data
import data_utils
import numpy as np
import importlib
importlib.reload(data)
# load data from pickle and npy files
metadata, idx_q, idx_a = data.load_data(PATH='datasets/IE/')
(trainX, trainY), (testX,
testY), (validX,
validY) = data_utils.split_dataset(idx_q, idx_a)
# parameters
xseq_len = trainX.shape[-1]
yseq_len = trainY.shape[-1]
xvocab_size = len(metadata['idx2w'])
yvocab_size = xvocab_size
emb_dim = 1024
import seq2seq_wrapper
model = seq2seq_wrapper.Seq2Seq(xseq_len=xseq_len,
yseq_len=yseq_len,
xvocab_size=xvocab_size,
yvocab_size=yvocab_size,
ckpt_path='ckpt/IE/',
loss_path='',
metadata=metadata,
emb_dim=emb_dim,