def __init__(self, vocab_size=10000, wordvec_size=650, hidden_size=650,
dropout_ratio=0.5):
V, D, H = vocab_size, wordvec_size, hidden_size
embed_W = (np.random.randn(V, D) / 100).astype('f')
lstm_Wx1 = (np.random.randn(D, 4 * H) / np.sqrt(D)).astype('f')
lstm_Wh1 = (np.random.randn(H, 4 * H) / np.sqrt(H)).astype('f')
lstm_b1 = np.zeros(4 * H).astype('f')
lstm_Wx2 = (np.random.randn(D, 4 * H) / np.sqrt(D)).astype('f')
lstm_Wh2 = (np.random.randn(H, 4 * H) / np.sqrt(H)).astype('f')
lstm_b2 = np.zeros(4 * H).astype('f')
affine_b = np.zeros(V).astype('f')
self.layers = [
TimeEmbedding(embed_W),
TimeDropout(dropout_ratio),
TimeLSTM(lstm_Wx1, lstm_Wh1, lstm_b1, stateful=True),
TimeDropout(dropout_ratio),
TimeLSTM(lstm_Wx2, lstm_Wh2, lstm_b2, stateful=True),
TimeDropout(dropout_ratio),
TimeAffine(embed_W.T, affine_b)
]
self.loss_layer = TimeSoftmaxWithLoss()
self.lstm_layers = [self.layers[2], self.layers[4]]
self.drop_layers = [self.layers[1], self.layers[3], self.layers[5]]
self.params = []
self.grads = []
for layer in self.layers:
self.params += layer.params
self.grads += layer.grads
class Seq2Seq:
def __init__(self, vocab_size, wordvec_size, hidden_size):
V, D, H = vocab_size, wordvec_size, hidden_size
self.encoder = Encoder(V, D, H)
self.decoder = Decoder(V, D, H)
self.loss_layer = TimeSoftmaxWithLoss()
self.params = []
self.grads = []
for layer in (self.encoder, self.decoder):
self.params += layer.params
self.grads += layer.grads
def forward(self, xs, ts):
decoder_xs = ts[:, :-1]
decoder_ts = ts[:, 1:]
h = self.encoder.forward(xs)
ys = self.decoder.forward(decoder_xs, h)
loss = self.loss_layer.forward(ys, decoder_ts)
return loss
def backward(self, dout=1):
dy = self.loss_layer.backward(dout)
dh = self.decoder.backward(dy)
dout = self.encoder.backward(dh)
return dout
def generate(self, xs, start_id, sample_size):
h = self.encoder.forward(xs)
sampled = self.decoder.generate(h, start_id, sample_size)
return sampled
def __init__(self, vocab_size=10000, wordvec_size=100, hidden_size=100):
V, D, H = vocab_size, wordvec_size, hidden_size
embed_W = (np.random.randn(V, D) / 100).astype('f')
lstm_Wx = (np.random.randn(D, 4 * H) / np.sqrt(D)).astype('f')
lstm_Wh = (np.random.randn(H, 4 * H) / np.sqrt(H)).astype('f')
lstm_b = np.zeros(4 * H).astype('f')
affine_W = (np.random.randn(H, V) / np.sqrt(H)).astype('f')
affine_b = np.zeros(V).astype('f')
self.layers = [
TimeEmbedding(embed_W),
TimeLSTM(lstm_Wx, lstm_Wh, lstm_b, stateful=True),
TimeAffine(affine_W, affine_b)
]
self.loss_layer = TimeSoftmaxWithLoss()
self.lstm_layer = self.layers[1]
self.params = []
self.grads = []
for layer in self.layers:
self.params += layer.params
self.grads += layer.grads
def __init__(self, vocab_size, word_vecs, hidden_size):
V, D, H = vocab_size, word_vecs, hidden_size
embed_W = (np.random.randn(V, D) / 100).astype('f')
rnn_Wx = (np.random.randn(D, H) / np.sqrt(D)).astype('f')
rnn_Wh = (np.random.randn(H, H) / np.sqrt(H)).astype('f')
rnn_b = np.zeros(H).astype('f')
affine_W = (np.random.randn(H, V) / np.sqrt(H)).astype('f')
affine_b = np.zeros(V).astype('f')
self.layers = [
TimeEmbedding(embed_W),
TimeRNN(rnn_Wx, rnn_Wh, rnn_b, stateful=True),
TimeAffine(affine_W, affine_b)
]
self.loss_layer = TimeSoftmaxWithLoss()
self.rnn_layer = self.layers[1]
self.params = []
self.grads = []
for layer in self.layers:
self.params += layer.params
self.grads += layer.grads
class RNNLM:
def __init__(self, vocab_size=10000, wordvec_size=100, hidden_size=100):
V, D, H = vocab_size, wordvec_size, hidden_size
embed_W = (np.random.randn(V, D) / 100).astype('f')
lstm_Wx = (np.random.randn(D, 4 * H) / np.sqrt(D)).astype('f')
lstm_Wh = (np.random.randn(H, 4 * H) / np.sqrt(H)).astype('f')
lstm_b = np.zeros(4 * H).astype('f')
affine_W = (np.random.randn(H, V) / np.sqrt(H)).astype('f')
affine_b = np.zeros(V).astype('f')
self.layers = [
TimeEmbedding(embed_W),
TimeLSTM(lstm_Wx, lstm_Wh, lstm_b, stateful=True),
TimeAffine(affine_W, affine_b)
]
self.loss_layer = TimeSoftmaxWithLoss()
self.lstm_layer = self.layers[1]
self.params = []
self.grads = []
for layer in self.layers:
self.params += layer.params
self.grads += layer.grads
def predict(self, xs):
for layer in self.layers:
xs = layer.forward(xs)
return xs
def forward(self, xs, ts):
ys = self.predict(xs)
loss = self.loss_layer.forward(ys, ts)
return loss
def backward(self, dout=1):
dout = self.loss_layer.backward(dout)
for layer in reversed(self.layers):
dout = layer.backward(dout)
return dout
def reset_state(self):
self.lstm_layer.reset_state()
def save_params(self, filename='weights/RNN_params.pkl'):
params = [param.astype(np.float16) for param in self.params]
with open(filename, 'wb') as f:
pickle.dump(params, f)
def load_params(self, filename='weights/RNN_params.pkl'):
print('load params... ', end='')
with open(filename, 'rb') as f:
params = pickle.load(f)
for i, param in enumerate(self.params):
param[...] = params[i].astype('f')
print('done.')
def __init__(self, vocab_size, wordvec_size, hidden_size):
V, D, H = vocab_size, wordvec_size, hidden_size
self.encoder = Encoder(V, D, H)
self.decoder = Decoder(V, D, H)
self.loss_layer = TimeSoftmaxWithLoss()
self.params = []
self.grads = []
for layer in (self.encoder, self.decoder):
self.params += layer.params
self.grads += layer.grads
class SimpleRNNLM:
def __init__(self, vocab_size, word_vecs, hidden_size):
V, D, H = vocab_size, word_vecs, hidden_size
embed_W = (np.random.randn(V, D) / 100).astype('f')
rnn_Wx = (np.random.randn(D, H) / np.sqrt(D)).astype('f')
rnn_Wh = (np.random.randn(H, H) / np.sqrt(H)).astype('f')
rnn_b = np.zeros(H).astype('f')
affine_W = (np.random.randn(H, V) / np.sqrt(H)).astype('f')
affine_b = np.zeros(V).astype('f')
self.layers = [
TimeEmbedding(embed_W),
TimeRNN(rnn_Wx, rnn_Wh, rnn_b, stateful=True),
TimeAffine(affine_W, affine_b)
]
self.loss_layer = TimeSoftmaxWithLoss()
self.rnn_layer = self.layers[1]
self.params = []
self.grads = []
for layer in self.layers:
self.params += layer.params
self.grads += layer.grads
def forward(self, xs, ts):
for layer in self.layers:
xs = layer.forward(xs)
loss = self.loss_layer.forward(xs, ts)
return loss
def backward(self, dout=1):
dout = self.loss_layer.backward(dout)
for layer in reversed(self.layers):
dout = layer.backward(dout)
return dout
def reset_state(self):
self.rnn_layer.reset_state()
class BetterRNNLM:
def __init__(self, vocab_size=10000, wordvec_size=650, hidden_size=650,
dropout_ratio=0.5):
V, D, H = vocab_size, wordvec_size, hidden_size
embed_W = (np.random.randn(V, D) / 100).astype('f')
lstm_Wx1 = (np.random.randn(D, 4 * H) / np.sqrt(D)).astype('f')
lstm_Wh1 = (np.random.randn(H, 4 * H) / np.sqrt(H)).astype('f')
lstm_b1 = np.zeros(4 * H).astype('f')
lstm_Wx2 = (np.random.randn(D, 4 * H) / np.sqrt(D)).astype('f')
lstm_Wh2 = (np.random.randn(H, 4 * H) / np.sqrt(H)).astype('f')
lstm_b2 = np.zeros(4 * H).astype('f')
affine_b = np.zeros(V).astype('f')
self.layers = [
TimeEmbedding(embed_W),
TimeDropout(dropout_ratio),
TimeLSTM(lstm_Wx1, lstm_Wh1, lstm_b1, stateful=True),
TimeDropout(dropout_ratio),
TimeLSTM(lstm_Wx2, lstm_Wh2, lstm_b2, stateful=True),
TimeDropout(dropout_ratio),
TimeAffine(embed_W.T, affine_b)
]
self.loss_layer = TimeSoftmaxWithLoss()
self.lstm_layers = [self.layers[2], self.layers[4]]
self.drop_layers = [self.layers[1], self.layers[3], self.layers[5]]
self.params = []
self.grads = []
for layer in self.layers:
self.params += layer.params
self.grads += layer.grads
def predict(self, xs, train_flg=False):
for layer in self.drop_layers:
layer.train_flg = train_flg
for layer in self.layers:
xs = layer.forward(xs)
return xs
def forward(self, xs, ts, train_flg=True):
ys = self.predict(xs, train_flg)
loss = self.loss_layer.forward(ys, ts)
return loss
def backward(self, dout=1):
dout = self.loss_layer.backward(dout)
for layer in reversed(self.layers):
dout = layer.backward(dout)
return dout
def reset_state(self):
for layer in self.lstm_layers:
layer.reset_state()
def save_params(self, filename='weights/better_RNN_params.pkl'):
params = [param.astype(np.float16) for param in self.params]
with open(filename, 'wb') as f:
pickle.dump(params, f)
def load_params(self, filename='weights/better_RNN_params.pkl'):
print('load params... ', end='')
with open(filename, 'rb') as f:
params = pickle.load(f)
for i, param in enumerate(self.params):
param[...] = params[i].astype('f')
print('done.')