def forward(self, inputs, hidden, genotype):
T, B = inputs.size(0), inputs.size(1) # time of step , batch size
if self.training:
x_mask = mask2d(B, inputs.size(2), keep_prob=1. - self.dropout_x)
h_mask = mask2d(B, hidden.size(2), keep_prob=1. - self.dropout_h)
else:
x_mask = h_mask = None
hidden = hidden[0]
hiddens = []
for t in range(T): # 进入darts cell search的cell函数
hidden = self.cell(inputs[t], hidden, x_mask, h_mask, genotype)
hiddens.append(hidden)
hiddens = torch.stack(hiddens)
return hiddens, hiddens[-1].unsqueeze(0)
def forward(self, inputs, hidden):
T, B = inputs.size(0), inputs.size(1)
if self.training:
x_mask = mask2d(B, inputs.size(2), keep_prob=1. - self.dropoutx)
h_mask = mask2d(B, hidden.size(2), keep_prob=1. - self.dropouth)
else:
x_mask = h_mask = None
hidden = hidden[0]
hiddens = []
for t in range(T):
hidden = self.cell(inputs[t], hidden, x_mask, h_mask)
hiddens.append(hidden)
hiddens = torch.stack(hiddens)
return hiddens, hiddens[-1].unsqueeze(0)
def forward(self, inputs, hidden):
T, B = inputs.size(0), inputs.size(1)
if self.training:
x_mask = mask2d(B, inputs.size(2), keep_prob=1. - self.dropoutx)
h_mask = mask2d(B, hidden.size(2), keep_prob=1. - self.dropouth)
else:
x_mask = h_mask = None
hidden = hidden[0]
hiddens = []
clipped_num = 0
max_clipped_norm = 0
# forward pass through time in the cell, T is the sequence length
for t in range(T):
"""
if hidden.max().data[0] > 1000 or inputs.max().data[0] > 1000:
model_logger.info("FORWARD IN CELL CLASS, EXPLODING IN step " + str(t))
model_logger.info("hidden max: " + str(hidden.max().data[0]))
model_logger.info("input max: " + str(inputs.max().data[0]))
"""
# main forward inside the cell
hidden = self.cell(inputs[t], hidden, x_mask, h_mask)
if self.handle_hidden_mode == 'NORM':
hidden_norms = hidden.norm(dim=-1)
max_norm = 25.0
if hidden_norms.data.max() > max_norm:
# Just directly use the torch slice operations
# in PyTorch v0.4.
#
# This workaround for PyTorch v0.3.1 does everything in numpy,
# because the PyTorch slicing and slice assignment is too
# flaky.
hidden_norms = hidden_norms.data.cpu().numpy()
clipped_num += 1
if hidden_norms.max() > max_clipped_norm:
max_clipped_norm = hidden_norms.max()
clip_select = hidden_norms > max_norm
clip_norms = hidden_norms[clip_select]
mask = np.ones(hidden.size())
normalizer = max_norm / clip_norms
normalizer = normalizer[:, np.newaxis]
mask[clip_select] = normalizer
hidden *= torch.autograd.Variable(
torch.FloatTensor(mask).cuda(), requires_grad=False)
# saving the hidden output for each step
hiddens.append(hidden)
if clipped_num > 0:
model_logger.info(
'clipped {} hidden states in one forward '.format(clipped_num))
model_logger.info(
'max clipped hidden state norm: {}'.format(max_clipped_norm))
# creating a tensor from the list of hiddens in order to have the elements stacked
hiddens = torch.stack(hiddens)
# return the stack of hidden outputs and the hidden output of the last step
return hiddens, hiddens[-1].unsqueeze(0)