def forward(self, inputs):
torch.embedding()
conv1 = self.conv1(inputs)
conv2 = self.conv2(conv1)
res = self.out(conv2)
return res
def forward(self, indices: Union[Tensor, PackedSequence], dim: int = -2) -> Union[Tensor, PackedSequence]:
if torch.is_tensor(indices):
return self.weight[:indices.size()[dim]]
batch_sizes = indices.batch_sizes.to(device=indices.data.device)
indices, _ = major_sizes_to_ptr(batch_sizes=batch_sizes)
return torch.embedding(weight=self.weight, indices=indices)
def embedding(self,
input,
weight,
padding_idx=None,
max_norm=None,
norm_type=2,
scale_grad_by_freq=False,
sparse=False):
input = input.contiguous()
if padding_idx is not None:
if padding_idx > 0:
assert padding_idx < weight.size(
0), 'Padding_idx must be within num_embeddings'
elif padding_idx < 0:
assert padding_idx >= -weight.size(
0), 'Padding_idx must be within num_embeddings'
padding_idx = weight.size(0) + padding_idx
elif padding_idx is None:
padding_idx = -1
if max_norm is not None:
with torch.no_grad():
torch.embedding_renorm_(weight, input, max_norm, norm_type)
return torch.embedding(weight, input, padding_idx, scale_grad_by_freq,
sparse)
def forward(self, position_ids):
max_pos_id = position_ids.max()
# update positional encoding if needed
if max_pos_id >= self._max_sequence_length:
self._max_sequence_length = max_pos_id + 1
self._build_pos_enc(
hidden_size=self._hidden_size,
max_sequence_length=self._max_sequence_length,
device=position_ids.device,
)
return torch.embedding(self.pos_enc, position_ids)
def forward(self, sentences: torch.Tensor) -> torch.Tensor:
batch, maxlen = sentences.size()
swapped_sentence = sentences.transpose(0, 1)
norm_input = torch.log_softmax(self.input, dim=0)
forward_emission = torch.embedding(swapped_sentence.reshape(-1), norm_input).reshape(maxlen, batch, self.num_state)
forward_transition = self.logsoftmax(self.transition.unsqueeze(0))
# forward
forwards = [forward_emission[0]]
# forwards = [self.tanh(forward_emission[0])]
for i in range(1, maxlen):
pre_forward = forwards[i - 1]
current_forward = self.bmv_log_product(forward_transition, pre_forward)
forwards.append(current_forward + forward_emission[i])
# current_forward = torch.matmul(forward_transition, pre_forward.unsqueeze(-1)).squeeze(-1)
# forwards.append(current_forward * forward_emission[i])
# backward
backward_emission = forward_emission.flip(dims=[0])
backward_transition = forward_transition.transpose(1, 2)
backwards = [backward_emission[0]]
for i in range(1, maxlen):
pre_backward = backwards[i - 1]
current_backward = self.bmv_log_product(backward_transition, pre_backward)
backwards.append(current_backward + backward_emission[i])
# current_backward = torch.matmul(backward_transition, pre_backward.unsqueeze(-1)).squeeze(-1)
# backwards.append(current_backward * backward_emission[i])
forwards = torch.stack(forwards, dim=0)
backwards = torch.stack(backwards[::-1], dim=0)
# nan_detection(forwards)
# nan_detection(backwards)
expected_count = forwards + backwards - forward_emission
expected_count = expected_count - torch.logsumexp(expected_count, dim=-1, keepdim=True)
# expected_count_debug = forwards * backwards / forward_emission
# expected_count1 = backwards[0].unsqueeze(0)
# expected_count2 = torch.matmul(forward_transition.unsqueeze(0), forwards[:-1].unsqueeze(-1)).squeeze(-1) * backwards[1:]
# expected_count = torch.cat([expected_count1, expected_count2], dim=0)
# expected_count = smoothing(expected_count)
# nan_detection(expected_count)
score = self.bmm_log_product(expected_count, self.logsoftmax(self.output.unsqueeze(0)))
# score = torch.matmul(expected_count, self.output.unsqueeze(0))
# nan_detection(score)
return score.transpose(0, 1)
def test_vq_st_gradient2():
inputs = torch.rand((2, 3, 5, 7), dtype=torch.float32, requires_grad=True)
codebook = torch.rand((11, 7), dtype=torch.float32, requires_grad=True)
codes, _ = vq_st(inputs, codebook)
indices = vq(inputs, codebook)
codes_torch = torch.embedding(codebook, indices, padding_idx=-1,
scale_grad_by_freq=False, sparse=False)
grad_output = torch.rand((2, 3, 5, 7), dtype=torch.float32)
grad_codebook, = torch.autograd.grad(codes, codebook,
grad_outputs=[grad_output])
grad_codebook_torch, = torch.autograd.grad(codes_torch, codebook,
grad_outputs=[grad_output])
# Gradient is the same as torch.embedding function
assert grad_codebook.size() == (11, 7)
assert np.allclose(grad_codebook.numpy(), grad_codebook_torch.numpy())
def test_emb_grad():
# Embedding vector gradient is the same as torch.embedding function
input = torch.rand((2, 3, 5, 7), dtype=torch.float32, requires_grad=True)
emb = torch.rand((11, 7), dtype=torch.float32, requires_grad=True)
output, latents = QuantizeVector.apply(input, emb)
output_torch = torch.embedding(emb, latents, padding_idx=-1,
scale_grad_by_freq=False, sparse=False)
grad_output = torch.rand((2, 3, 5, 7), dtype=torch.float32)
grad_emb, = torch.autograd.grad(output, emb,
grad_outputs=[grad_output])
grad_emb_torch, = torch.autograd.grad(output_torch, emb,
grad_outputs=[grad_output])
assert grad_emb.size() == (11, 7), \
'Embedding gradient shape does not match the embedding space.'
assert np.allclose(grad_emb.numpy(), grad_emb_torch.numpy()), \
'Embedding gradients are not equal to the torch.embedding function.'
def forward(self, idx):
"""pull emb from server"""
with torch.no_grad():
bsz, slen = idx.size()
cpu_idx = idx.cpu()
unique_idx = torch.unique(cpu_idx)
unique_emb = self.client.pull(self.name, unique_idx)
gpu_emb = unique_emb.to(idx.device).detach_().requires_grad_(True)
idx_mapping = {i.item(): j for j, i in enumerate(unique_idx)}
mapped_idx = torch.zeros((bsz, slen), dtype=torch.long)
for i in range(bsz):
for j in range(slen):
mapped_idx[i][j] = idx_mapping[cpu_idx[i][j].item()]
# emb = torch.index_select(gpu_emb, 0, mapped_idx.cuda().view(-1)).view(bsz, slen, -1)
emb = torch.embedding(gpu_emb, mapped_idx.cuda())
# print('emb norm: {:.3f}, dtype: {}'.format(torch.norm(emb.data), emb.dtype))
if self.training:
self.trace.append((unique_idx, gpu_emb))
return emb
def forward(self, sentences: torch.Tensor, length: torch.Tensor) -> torch.Tensor:
batch, maxlen = sentences.size()
swapped_sentence = sentences.transpose(0, 1)
norm_embeding = torch.log_softmax(self.input, dim=0)
forward_emission = self.logsoftmax(torch.embedding(swapped_sentence.view(-1), norm_embeding).view(maxlen, batch, self.num_state))
forward_transition = self.logsoftmax(self.transition).unsqueeze(0)
# forward
forwards = [forward_emission[0]]
# forwards = [self.tanh(forward_emission[0])]
for i in range(1, maxlen):
pre_forward = forwards[i - 1]
current_forward = self.bmv_log_product(forward_transition, pre_forward)
forwards.append(current_forward + forward_emission[i])
# current_forward = torch.matmul(forward_transition, pre_forward.unsqueeze(-1)).squeeze(-1)
# forwards.append(current_forward * forward_emission[i])
# shape [batch, dim]
hidden_states = torch.stack(forwards)[length-1, torch.arange(batch), :]
# shape [batch, label]
score = self.bmv_log_product(self.logsoftmax(self.output.unsqueeze(0)), hidden_states)
# score = torch.matmul(expected_count, self.output.unsqueeze(0))
# nan_detection(score)
return score
def Fembedding(input, weight, padding_idx=None, max_norm=None, norm_type=2,
scale_grad_by_freq=False, sparse=False):
r"""A simple lookup table that looks up embeddings in a fixed dictionary and size.
This module is often used to retrieve word embeddings using indices.
The input to the module is a list of indices, and the embedding matrix,
and the output is the corresponding word embeddings.
See :class:`torch.nn.Embedding` for more details.
Args:
input (LongTensor): Tensor containing indices into the embedding matrix
weight (Tensor): The embedding matrix
Number of rows should correspond to the maximum possible index + 1,
number of columns is the embedding size
padding_idx (int, optional): If given, pads the output with the embedding vector at :attr:`padding_idx`
(initialized to zeros) whenever it encounters the index.
max_norm (float, optional): If given, will renormalize the embedding vectors to have a norm lesser than
this before extracting. Note: this will modify :attr:`weight` in-place.
norm_type (float, optional): The p of the p-norm to compute for the max_norm option. Default ``2``.
scale_grad_by_freq (boolean, optional): if given, this will scale gradients by the inverse of frequency of
the words in the mini-batch. Default ``False``.
sparse (bool, optional): if ``True``, gradient w.r.t. :attr:`weight` will be a sparse tensor. See Notes under
:class:`torch.nn.Embedding` for more details regarding sparse gradients.
Shape:
- Input: LongTensor of arbitrary shape containing the indices to extract
- Weight: Embedding matrix of floating point type with shape `(V, embedding_dim)`,
where V = maximum index + 1 and embedding_dim = the embedding size
- Output: `(*, embedding_dim)`, where `*` is the input shape
Examples::
>>> # a batch of 2 samples of 4 indices each
>>> input = torch.tensor([[1,2,4,5],[4,3,2,9]])
>>> # an embedding matrix containing 10 tensors of size 3
>>> embedding_matrix = torch.rand(10, 3)
>>> F.embedding(input, embedding_matrix)
tensor([[[ 0.8490, 0.9625, 0.6753],
[ 0.9666, 0.7761, 0.6108],
[ 0.6246, 0.9751, 0.3618],
[ 0.4161, 0.2419, 0.7383]],
[[ 0.6246, 0.9751, 0.3618],
[ 0.0237, 0.7794, 0.0528],
[ 0.9666, 0.7761, 0.6108],
[ 0.3385, 0.8612, 0.1867]]])
>>> # example with padding_idx
>>> weights = torch.rand(10, 3)
>>> weights[0, :].zero_()
>>> embedding_matrix = weights
>>> input = torch.tensor([[0,2,0,5]])
>>> F.embedding(input, embedding_matrix, padding_idx=0)
tensor([[[ 0.0000, 0.0000, 0.0000],
[ 0.5609, 0.5384, 0.8720],
[ 0.0000, 0.0000, 0.0000],
[ 0.6262, 0.2438, 0.7471]]])
"""
if padding_idx is not None:
if padding_idx > 0:
assert padding_idx < weight.size(0), 'Padding_idx must be within num_embeddings'
elif padding_idx < 0:
assert padding_idx >= -weight.size(0), 'Padding_idx must be within num_embeddings'
padding_idx = weight.size(0) + padding_idx
elif padding_idx is None:
padding_idx = -1
if max_norm is not None:
# `embedding_renorm_` will call .contiguous() on input anyways, so we
# call it here and take advantage of the improved locality in the
# `embedding` call below too.
input = input.contiguous()
with torch.no_grad():
torch.embedding_renorm_(weight, input, max_norm, norm_type)
return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
def function_hook(input, weight, *args, **kwargs):
return torch.embedding(weight, input)
def forward(self, position_ids):
return torch.embedding(self.pos_enc, position_ids)
def RNN(testloader, lines_test):
act_bit = 8
model = torch.load('lstm_model.pt')
embed_weight = model.embed.weight.data.numpy()
lstm_ih_weight = model.rnn.weight_ih_l0.data.numpy().T
lstm_hh_weight = model.rnn.weight_hh_l0.data.numpy().T
lstm_ih_bias = model.rnn.bias_ih_l0.data.numpy()
lstm_hh_bias = model.rnn.bias_hh_l0.data.numpy()
w_ii = lstm_ih_weight[:, :50]
w_if = lstm_ih_weight[:, 50:100]
w_ig = lstm_ih_weight[:, 100:150]
w_io = lstm_ih_weight[:, 150:]
w_hi = lstm_hh_weight[:, :50]
w_hf = lstm_hh_weight[:, 50:100]
w_hg = lstm_hh_weight[:, 100:150]
w_ho = lstm_hh_weight[:, 150:]
b_ii = lstm_ih_bias[:50]
b_if = lstm_ih_bias[50:100]
b_ig = lstm_ih_bias[100:150]
b_io = lstm_ih_bias[150:]
b_hi = lstm_hh_bias[:50]
b_hf = lstm_hh_bias[50:100]
b_hg = lstm_hh_bias[100:150]
b_ho = lstm_hh_bias[150:]
fc_weight = model.fc.weight.data.numpy().T
fc_bias = model.fc.bias.data.numpy()
# scale_w_ii = (max(abs(np.max(w_ii)), abs(np.min(w_ii))) / 127)
# scale_w_if = (max(abs(np.max(w_if)), abs(np.min(w_if))) / 127)
# scale_w_ig = (max(abs(np.max(w_ig)), abs(np.min(w_ig))) / 127)
# scale_w_io = (max(abs(np.max(w_io)), abs(np.min(w_io))) / 127)
# scale_w_hi = (max(abs(np.max(w_hi)), abs(np.min(w_hi))) / 127)
# scale_w_hf = (max(abs(np.max(w_hf)), abs(np.min(w_hf))) / 127)
# scale_w_hg = (max(abs(np.max(w_hg)), abs(np.min(w_hg))) / 127)
# scale_w_ho = (max(abs(np.max(w_ho)), abs(np.min(w_ho))) / 127)
scale_ih_weight = (
max(abs(np.max(lstm_ih_weight)), abs(np.min(lstm_ih_weight))) /
(2**(act_bit - 1) - 1))
scale_hh_weight = (
max(abs(np.max(lstm_hh_weight)), abs(np.min(lstm_hh_weight))) /
(2**(act_bit - 1) - 1))
scale_fc_weight = (max(abs(np.max(fc_weight)), abs(np.min(fc_weight))) /
(2**(act_bit - 1) - 1))
# w_ii = np.round(w_ii / scale_w_ii)
# w_if = np.round(w_if / scale_w_if)
# w_ig = np.round(w_ig / scale_w_ig)
# w_io = np.round(w_io / scale_w_io)
# w_hi = np.round(w_hi / scale_w_hi)
# w_hf = np.round(w_hf / scale_w_hf)
# w_hg = np.round(w_hg / scale_w_hg)
# w_ho = np.round(w_ho / scale_w_ho)
# w_ii = np.round(w_ii / scale_ih_weight)
# w_if = np.round(w_if / scale_ih_weight)
# w_ig = np.round(w_ig / scale_ih_weight)
# w_io = np.round(w_io / scale_ih_weight)
# w_hi = np.round(w_hi / scale_hh_weight)
# w_hf = np.round(w_hf / scale_hh_weight)
# w_hg = np.round(w_hg / scale_hh_weight)
# w_ho = np.round(w_ho / scale_hh_weight)
lstm_ih_weight = np.round(lstm_ih_weight / scale_ih_weight)
lstm_hh_weight = np.round(lstm_hh_weight / scale_hh_weight)
fc_weight = np.round(fc_weight / scale_fc_weight)
# w_ii_mapped = fc_mapping(w_ii)
# w_if_mapped = fc_mapping(w_if)
# w_ig_mapped = fc_mapping(w_ig)
# w_io_mapped = fc_mapping(w_io)
# w_hi_mapped = fc_mapping(w_hi)
# w_hf_mapped = fc_mapping(w_hf)
# w_hg_mapped = fc_mapping(w_hg)
# w_ho_mapped = fc_mapping(w_ho)
weight_ih_mapped = fc_mapping(lstm_ih_weight)
weight_hh_mapped = fc_mapping(lstm_hh_weight)
fc_weight_mapped = fc_mapping(fc_weight)
input_dim, hidden_dim = w_ii.shape
total = 0
correct = 0
for data in testloader:
line = lines_test[total][2:-1]
total += 1
inputs, labels = data
embed = np.squeeze(
torch.embedding(torch.tensor(embed_weight),
inputs).numpy())[:, np.newaxis, :]
inputs = inputs.numpy()
labels = labels.numpy()
_, time_step = inputs.shape
hidden_prev = np.zeros((1, hidden_dim))
c_prev = np.zeros((1, hidden_dim))
output_rnn = np.zeros((time_step, hidden_dim))
scale_input = max(abs(np.max(embed)), abs(
np.min(embed))) / (2**(act_bit - 1) - 1)
# b_ii_q = b_ii / scale_w_ii / scale2bit(scale_input)
# b_if_q = b_if / scale_w_if / scale2bit(scale_input)
# b_ig_q = b_ig / scale_w_ig / scale2bit(scale_input)
# b_io_q = b_io / scale_w_io / scale2bit(scale_input)
# b_hi_q = b_hi / scale_w_hi / scale2bit(scale_input)
# b_hf_q = b_hf / scale_w_hf / scale2bit(scale_input)
# b_hg_q = b_hg / scale_w_hg / scale2bit(scale_input)
# b_ho_q = b_ho / scale_w_ho / scale2bit(scale_input)
# b_ii_q = b_ii / scale_ih_weight / scale2bit(scale_input)
# b_if_q = b_if / scale_ih_weight / scale2bit(scale_input)
# b_ig_q = b_ig / scale_ih_weight / scale2bit(scale_input)
# b_io_q = b_io / scale_ih_weight / scale2bit(scale_input)
# b_hi_q = b_hi / scale_hh_weight / scale2bit(scale_input)
# b_hf_q = b_hf / scale_hh_weight / scale2bit(scale_input)
# b_hg_q = b_hg / scale_hh_weight / scale2bit(scale_input)
# b_ho_q = b_ho / scale_hh_weight / scale2bit(scale_input)
lstm_ih_bias_q = lstm_ih_bias / scale_ih_weight / (scale_input)
lstm_hh_bias_q = lstm_hh_bias / scale_hh_weight / (scale_input)
embed = np.round(embed / (scale_input))
for t in range(time_step):
scale_hidden = max(abs(
np.max(hidden_prev)), abs(
np.min(hidden_prev))) / (2**(act_bit - 1) - 1)
scale_c = max(abs(np.max(c_prev)), abs(
np.min(c_prev))) / (2**(act_bit - 1) - 1)
if scale_hidden == 0:
scale_hidden = 0.001
if scale_c == 0:
scale_c = 0.001
hidden_prev = np.round(hidden_prev / (scale_hidden))
# c_prev = np.round(c_prev/ scale2bit(scale_c / (2**(act_bit-1)-1)))
# i = sigmoid(fc_int(embed[t], scale_input, w_ii, w_ii_mapped, scale_w_ii, b_ii_q, 8, True) \
# + fc_int(hidden_prev, scale_hidden, w_hi, w_hi_mapped, scale_w_hi, b_hi_q, 8, True))
# f = sigmoid(fc_int(embed[t], scale_input, w_if, w_if_mapped, scale_w_if, b_if_q, 8, True) \
# + fc_int(hidden_prev, scale_hidden, w_hf, w_hf_mapped, scale_w_hf, b_hf_q, 8, True))
# g = tanh(fc_int(embed[t], scale_input, w_ig, w_ig_mapped, scale_w_ig, b_ig_q, 8, True) \
# + fc_int(hidden_prev, scale_hidden, w_hg, w_hg_mapped, scale_w_hg, b_hg_q, 8, True))
# o = sigmoid(fc_int(embed[t], scale_input, w_io, w_io_mapped, scale_w_io, b_io_q, 8, True) \
# + fc_int(hidden_prev, scale_hidden, w_ho, w_ho_mapped, scale_w_ho, b_ho_q, 8, True))
tmp = fc_int(embed[t], scale_input, lstm_ih_weight, weight_ih_mapped, scale_ih_weight, lstm_ih_bias_q, 8, True) \
+ fc_int(hidden_prev, scale_hidden, lstm_hh_weight, weight_hh_mapped, scale_hh_weight, lstm_hh_bias_q, 8, True)
i = sigmoid(tmp[:, :50])
f = sigmoid(tmp[:, 50:100])
g = tanh(tmp[:, 100:150])
o = sigmoid(tmp[:, 150:])
# i = sigmoid(fc_int(embed[t], scale_input, w_ii, w_ii_mapped, scale_ih_weight, b_ii_q, 8, True) \
# + fc_int(hidden_prev, scale_hidden, w_hi, w_hi_mapped, scale_hh_weight, b_hi_q, 8, True))
# f = sigmoid(fc_int(embed[t], scale_input, w_if, w_if_mapped, scale_ih_weight, b_if_q, 8, True) \
# + fc_int(hidden_prev, scale_hidden, w_hf, w_hf_mapped, scale_hh_weight, b_hf_q, 8, True))
# g = tanh(fc_int(embed[t], scale_input, w_ig, w_ig_mapped, scale_ih_weight, b_ig_q, 8, True) \
# + fc_int(hidden_prev, scale_hidden, w_hg, w_hg_mapped, scale_hh_weight, b_hg_q, 8, True))
# o = sigmoid(fc_int(embed[t], scale_input, w_io, w_io_mapped, scale_ih_weight, b_io_q, 8, True) \
# + fc_int(hidden_prev, scale_hidden, w_ho, w_ho_mapped, scale_hh_weight, b_ho_q, 8, True))
c = f * c_prev + i * g
output_rnn[t] = o * tanh(c)
hidden_prev = output_rnn[t][np.newaxis, :]
c_prev = c
scale_fc = max(abs(np.max(output_rnn)), abs(
np.min(output_rnn))) / (2**(act_bit - 1) - 1)
if scale_fc == 0:
scale_fc = 0.001
fc_bias_q = fc_bias / scale_fc_weight / (scale_fc)
output_rnn = np.round(output_rnn / (scale_fc))
output_fc = fc_int(output_rnn, scale_fc, fc_weight, fc_weight_mapped,
scale_fc_weight, fc_bias_q, 2, True)
# output_fc = fc(output_rnn, fc_weight, fc_weight_mapped) + fc_bias
outputs = np.mean(output_fc, axis=0)
prediction = np.argmax(outputs)
print(line)
print('label=' + str(int(data[-1][0])) + ', prediction=' +
str(prediction) + '\n')
if prediction == labels:
correct += 1
accuracy = correct / total
print(accuracy)
def __init__(self, num_user, num_movies):
super().__init__()
self.user_embed = nn.embedding(num_user, 32)
self.movie_embed = nn.embedding(num_movies, 32)
self.out = nn.Linear(64, 1)
self.step_scheduler_after = 'epoch'
def forward(ctx, inputs, weights):
inputs = inputs.contiguous()
ctx.save_for_backward(inputs)
ctx.num_weights = weights.size(0)
return torch.embedding(weights, inputs)
def RNN(testloader, lines_test):
act_bit = 8
model = torch.load('rnn_model.pt')
embed_weight = model.embed.weight.data.numpy()
rnn_ih_weight = model.rnn.weight_ih_l0.data.numpy().T
rnn_hh_weight = model.rnn.weight_hh_l0.data.numpy().T
rnn_ih_bias = model.rnn.bias_ih_l0.data.numpy()
rnn_hh_bias = model.rnn.bias_hh_l0.data.numpy()
fc_weight = model.fc.weight.data.numpy().T
fc_bias = model.fc.bias.data.numpy()
scale_ih_weight = (max(abs(np.max(rnn_ih_weight)), abs(np.min(rnn_ih_weight))) / (2**(act_bit-1)-1))
scale_hh_weight = (max(abs(np.max(rnn_hh_weight)), abs(np.min(rnn_hh_weight))) / (2**(act_bit-1)-1))
scale_fc_weight = (max(abs(np.max(fc_weight)), abs(np.min(fc_weight))) / (2**(act_bit-1)-1))
rnn_ih_weight = np.round(rnn_ih_weight / scale_ih_weight)
rnn_hh_weight = np.round(rnn_hh_weight / scale_hh_weight)
fc_weight = np.round(fc_weight / scale_fc_weight)
rnn_ih_weight_mapped = fc_mapping(rnn_ih_weight)
rnn_hh_weight_mapped = fc_mapping(rnn_hh_weight)
fc_weight_mapped = fc_mapping(fc_weight)
# fc_bias = (fc_bias / scale_fc_weight / scale2bit(scale_input))
input_dim, hidden_dim = rnn_ih_weight.shape
total = 0
correct = 0
for data in testloader:
line = lines_test[total][2:-1]
total += 1
inputs, labels = data
embed = np.squeeze(torch.embedding(torch.tensor(embed_weight), inputs).numpy())[:,np.newaxis,:]
inputs = inputs.numpy()
labels = labels.numpy()
_, time_step = inputs.shape
hidden_prev = np.zeros((1, hidden_dim))
output_rnn = np.zeros((time_step, hidden_dim))
scale_input = max(abs(np.max(embed)), abs(np.min(embed))) / (2**(act_bit-1)-1)
embed = np.round(embed/ (scale_input))
rnn_ih_bias_q = (rnn_ih_bias / scale_ih_weight / (scale_input))
rnn_hh_bias_q = (rnn_hh_bias / scale_hh_weight / (scale_input))
for i in range(time_step):
scale_hidden = max(abs(np.max(hidden_prev)), abs(np.min(hidden_prev))) / (2**(act_bit-1)-1)
if scale_hidden == 0:
scale_hidden = 0.001
hidden_prev = np.round(hidden_prev/ (scale_hidden))
hidden = fc_int(embed[i], scale_input, rnn_ih_weight, rnn_ih_weight_mapped, scale_ih_weight, rnn_ih_bias_q, 8, True) \
+ fc_int(hidden_prev, scale_hidden, rnn_hh_weight, rnn_hh_weight_mapped, scale_hh_weight, rnn_hh_bias_q, 8, True)
# hidden = fc(embed[i], rnn_ih_weight, rnn_ih_weight_mapped) + fc(hidden, rnn_hh_weight, rnn_hh_weight_mapped) + rnn_ih_bias + rnn_hh_bias
output_rnn[i] = tanh(hidden)
hidden_prev = output_rnn[i][np.newaxis,:]
scale_fc = max(abs(np.max(output_rnn)), abs(np.min(output_rnn))) / (2**(act_bit-1)-1)
if scale_fc == 0:
scale_fc = 0.001
output_rnn = np.round(output_rnn/ (scale_fc))
fc_bias_q = (fc_bias / scale_fc_weight / (scale_fc))
output_fc = fc_int(output_rnn, scale_fc, fc_weight, fc_weight_mapped, scale_fc_weight, fc_bias_q, 2, True)
# output_fc = fc(output_rnn, fc_weight, fc_weight_mapped) + fc_bias
outputs = np.mean(output_fc, axis=0)
prediction = np.argmax(outputs)
print(line)
print('label='+str(int(data[-1][0]))+', prediction='+str(prediction)+'\n')
if prediction == labels:
correct += 1
accuracy = correct / total
print(accuracy)
def forward(self,
_input,
use_pos_encods=False,
skip_slots=False,
slot_mask=None,
slot_vals=None,
slot_lengths=None):
if isinstance(_input, dict):
_input = _input[DATA_GLOVE]
embed_input = _input.clamp(min=0, max=self.num_words - 1)
embeds = self.embeddings(embed_input)
if not skip_slots and self.use_slot_embeddings:
slot_input = (self.slot_start_index - _input).clamp(
min=0, max=self.num_slots - 1)
slot_embeds = self.slot_embeddings(slot_input)
is_slot_embed = (_input <= self.slot_start_index).float()
if self.use_slot_value_embeddings:
if slot_mask is not None and slot_vals is not None and slot_lengths is not None:
slot_value_embeddings = self.slot_value_encoder(
slot_vals=slot_vals,
slot_lengths=slot_lengths,
embedding_module=self)
step_size = torch.arange(start=0,
end=slot_mask.size(0),
dtype=slot_mask.dtype,
device=slot_mask.device
) * slot_value_embeddings.shape[1]
extended_slot_mask = (slot_mask -
1) * (slot_mask < 0).long()
extended_slot_mask = (extended_slot_mask +
step_size.unsqueeze(dim=-1)).view(-1)
slot_value_embeddings = torch.embedding(
slot_value_embeddings.view(
-1, slot_value_embeddings.size(-1)),
extended_slot_mask).view(
slot_mask.size(0), slot_mask.size(1),
slot_value_embeddings.size(-1))
slot_value_gate = self.gated_slot_embedding_layer(
slot_embeds)
slot_embeds = slot_embeds + slot_value_gate * slot_value_embeddings
elif slot_vals is not None or slot_lengths is not None:
print(
"[#] WARNING: At least one of the slot embedding values were None, although another was given."
)
print("\tSlot mask: %s, slot vals: %s, slot lengths: %s" %
("None" if
(slot_mask is None) else "Not none", "None" if
(slot_vals is None) else "Not none", "None" if
(slot_lengths is None) else "Not none"))
if use_pos_encods and len(
_input.shape
) == 2 and self.positional_embedding_factor != 0:
pos_indices = self.generate_pos_indices(
slot_input, is_slot_embed)
slot_embeds = slot_embeds + self.positional_embedding_factor * self.positional_embeddings(
pos_indices)
is_slot_embed = is_slot_embed.view(*(list(_input.shape) + [1]))
embeds = (1 - is_slot_embed) * embeds + is_slot_embed * slot_embeds
embeds = self.embedding_dropout(embeds)
return embeds
