def forward(self, inputs, hidden, genotype): T, B = inputs.size(0), inputs.size(1) # time of step , batch size if self.training: x_mask = mask2d(B, inputs.size(2), keep_prob=1. - self.dropout_x) h_mask = mask2d(B, hidden.size(2), keep_prob=1. - self.dropout_h) else: x_mask = h_mask = None hidden = hidden[0] hiddens = [] for t in range(T): # 进入darts cell search的cell函数 hidden = self.cell(inputs[t], hidden, x_mask, h_mask, genotype) hiddens.append(hidden) hiddens = torch.stack(hiddens) return hiddens, hiddens[-1].unsqueeze(0)
def forward(self, inputs, hidden): T, B = inputs.size(0), inputs.size(1) if self.training: x_mask = mask2d(B, inputs.size(2), keep_prob=1. - self.dropoutx) h_mask = mask2d(B, hidden.size(2), keep_prob=1. - self.dropouth) else: x_mask = h_mask = None hidden = hidden[0] hiddens = [] for t in range(T): hidden = self.cell(inputs[t], hidden, x_mask, h_mask) hiddens.append(hidden) hiddens = torch.stack(hiddens) return hiddens, hiddens[-1].unsqueeze(0)
def forward(self, inputs, hidden): T, B = inputs.size(0), inputs.size(1) if self.training: x_mask = mask2d(B, inputs.size(2), keep_prob=1. - self.dropoutx) h_mask = mask2d(B, hidden.size(2), keep_prob=1. - self.dropouth) else: x_mask = h_mask = None hidden = hidden[0] hiddens = [] clipped_num = 0 max_clipped_norm = 0 # forward pass through time in the cell, T is the sequence length for t in range(T): """ if hidden.max().data[0] > 1000 or inputs.max().data[0] > 1000: model_logger.info("FORWARD IN CELL CLASS, EXPLODING IN step " + str(t)) model_logger.info("hidden max: " + str(hidden.max().data[0])) model_logger.info("input max: " + str(inputs.max().data[0])) """ # main forward inside the cell hidden = self.cell(inputs[t], hidden, x_mask, h_mask) if self.handle_hidden_mode == 'NORM': hidden_norms = hidden.norm(dim=-1) max_norm = 25.0 if hidden_norms.data.max() > max_norm: # Just directly use the torch slice operations # in PyTorch v0.4. # # This workaround for PyTorch v0.3.1 does everything in numpy, # because the PyTorch slicing and slice assignment is too # flaky. hidden_norms = hidden_norms.data.cpu().numpy() clipped_num += 1 if hidden_norms.max() > max_clipped_norm: max_clipped_norm = hidden_norms.max() clip_select = hidden_norms > max_norm clip_norms = hidden_norms[clip_select] mask = np.ones(hidden.size()) normalizer = max_norm / clip_norms normalizer = normalizer[:, np.newaxis] mask[clip_select] = normalizer hidden *= torch.autograd.Variable( torch.FloatTensor(mask).cuda(), requires_grad=False) # saving the hidden output for each step hiddens.append(hidden) if clipped_num > 0: model_logger.info( 'clipped {} hidden states in one forward '.format(clipped_num)) model_logger.info( 'max clipped hidden state norm: {}'.format(max_clipped_norm)) # creating a tensor from the list of hiddens in order to have the elements stacked hiddens = torch.stack(hiddens) # return the stack of hidden outputs and the hidden output of the last step return hiddens, hiddens[-1].unsqueeze(0)