Пример #1
0
 def _test_master_maintain_logic_experiment(self):
     for _t in range(100):
         self.ffp16_optim.zero_grad()
         for pair in self.data_set:
             yy_pred = (pair[0].mm(self.pparameters[0]).clamp(min=0).mm(
                 self.pparameters[1]))
             lloss = (yy_pred - pair[1]).pow(2).sum()
             lloss = lloss / len(self.data_set)
             with fp16_optimizer.scale_loss(
                     lloss, self.ffp16_optim,
                     delay_unscale=False) as scaled_loss:
                 scaled_loss.backward()
         # Run backprop
         # Check for overflow
         self.ffp16_optim.step()
Пример #2
0
 def _test_memory_efficient_logic_exp(self):
     for _t in range(100):
         self.MEoptim.zero_grad()
         for pair in self.data_set:
             y_pred = (pair[0].mm(self.parameters[0]).clamp(min=0).mm(
                 self.parameters[1]))
             loss = (y_pred - pair[1]).pow(2).sum()
             loss = loss / len(self.data_set)
             with fp16_optimizer.scale_loss(
                     loss, self.MEoptim,
                     delay_unscale=False) as scaled_loss:
                 scaled_loss.backward()
         # Run backprop
         # Check for overflow
         self.MEoptim.loss_scaler.check_overflow(self.MEoptim.param_groups)
         # If no overflow, unscale grad and update as usual
         if not self.MEoptim.loss_scaler.is_overflow:
             self.MEoptim.loss_scaler.unscale_grads(
                 self.MEoptim.param_groups)
             self.MEoptim.is_scaled = False
             self.MEoptim.inner_optimizer.step()
         """ if use the following code, weights can have slight abrevations.