Z = torch.sigmoid((X @ W_xz) + (H @ W_hz) + b_z)
R = torch.sigmoid((X @ W_xr) + (H @ W_hr) + b_r)
H_tilda = torch.tanh((X @ W_xh) + ((R * H) @ W_hh) + b_h)
H = Z * H + (1 - Z) * H_tilda
Y = H @ W_hq + b_q
outputs.append(Y)
return torch.cat(outputs, dim=0), (H, )
# Hyperparameters
batch_size, num_steps = 32, 35
train_iter, vocab = d2l.load_data_time_machine(batch_size, num_steps)
vocab_size, num_hiddens, device = len(vocab), 256, d2l.try_gpu()
num_epochs, lr = 500, 1
model = d2l.RNNModelScratch(len(vocab), num_hiddens, device, get_params,
init_gru_state, gru)
print('scratch model')
d2l.train_ch8(model, train_iter, vocab, lr, num_epochs, device)
num_inputs = vocab_size
gru_layer = nn.GRU(num_inputs, num_hiddens)
model = d2l.RNNModel(gru_layer, len(vocab))
model = model.to(device)
print('concise model')
# d2l.train_ch8(model, train_iter, vocab, lr, num_epochs, device)
"""
outputs and error look good
# params = [W_xz, W_hz, b_z, W_xh, W_hh, b_h, W_hq, b_q]
def gru_no_reset(inputs, state, params):
W_xz, W_hz, b_z, W_xh, W_hh, b_h, W_hq, b_q = params
H, = state
return params
def init_rnn_state(batch_size, num_hiddens, device):
return (torch.zeros((batch_size, num_hiddens), device=device), )
def rnn(inputs, state, params):
# Here `inputs` shape: (`num_steps`, `batch_size`, `vocab_size`)
W_xh, W_hh, b_h, W_hq, b_q = params
H, = state
outputs = []
# Shape of `X`: (`batch_size`, `vocab_size`)
for X in inputs:
H = torch.tanh(torch.mm(X, W_xh) + torch.mm(H, W_hh) + b_h)
Y = torch.mm(H, W_hq) + b_q
outputs.append(Y)
return torch.cat(outputs, dim=0), (H,)
vocab_size, num_hiddens, device = len(vocab), 256, d2l.try_gpu()
num_epochs, lr = 500, 1
# Test GRU
model_gru = d2l.RNNModelScratch(len(vocab), num_hiddens, device, get_params_gru,
#init_gru_state, gru)
d2l.train_ch8(model_gru, train_iter, vocab, lr, num_epochs, device)
# Test RNN
model_rnn = d2l.RNNModelScratch(len(vocab), num_hiddens, device, get_params_rnn,
init_rnn_state, rnn)
d2l.train_ch8(model_rnn, train_iter, vocab, lr, num_epochs, device)