def forward(self, data): if self.training_mode not in self._fn_forward: symb_in = make_tensor_or_tensors(data, 'X') symb_out = self.symb_forward(symb_in) self._fn_forward[self.training_mode] = df.th.function( inputs=aslist(symb_in), outputs=symb_out) return self._fn_forward[self.training_mode](*aslist(data))
def accumulate_gradients(self, data_in, data_tgt, loss): if self.training_mode not in self._fn_accum_grads: symb_in = make_tensor_or_tensors(data_in, 'X') symb_tgt = make_tensor_or_tensors(data_tgt, 'T') symb_out = self.symb_forward(symb_in) symb_err = loss.symb_forward(symb_out, symb_tgt) params, grads = self.unique_parameters() symb_grads = df.th.grad(cost=symb_err, wrt=params) grads_updates = [(grad, grad + symb_grad) for grad, symb_grad in zip(grads, symb_grads)] self._fn_accum_grads[self.training_mode] = df.th.function( inputs=aslist(symb_in) + aslist(symb_tgt), outputs=symb_err, updates=grads_updates ) args = aslist(data_in) + aslist(data_tgt) return self._fn_accum_grads[self.training_mode](*args)
def forward(self, data): if self.training_mode not in self._fn_forward: symb_in = make_tensor_or_tensors(data, 'X') symb_out = self.symb_forward(symb_in) self._fn_forward[self.training_mode] = df.th.function( inputs=aslist(symb_in), outputs=symb_out ) return self._fn_forward[self.training_mode](*aslist(data))
def accumulate_gradients(self, data_in, data_tgt, loss): if self.training_mode not in self._fn_accum_grads: symb_in = make_tensor_or_tensors(data_in, 'X') symb_tgt = make_tensor_or_tensors(data_tgt, 'T') symb_out = self.symb_forward(symb_in) symb_err = loss.symb_forward(symb_out, symb_tgt) params, grads = self.unique_parameters() symb_grads = df.th.grad(cost=symb_err, wrt=params) grads_updates = [(grad, grad + symb_grad) for grad, symb_grad in zip(grads, symb_grads)] self._fn_accum_grads[self.training_mode] = df.th.function( inputs=aslist(symb_in) + aslist(symb_tgt), outputs=symb_err, updates=grads_updates) args = aslist(data_in) + aslist(data_tgt) return self._fn_accum_grads[self.training_mode](*args)
def accumulate_statistics(self, data_in): if self.training_mode not in self._fn_accum_stats: symb_in = make_tensor_or_tensors(data_in, 'X') # Call forward once so it can compute some variables it'll actually # use in the stat updates collection. self.symb_forward(symb_in) stat_updates = self.get_stat_updates() if not stat_updates: # If there's no layer collecting statistics, we don't need to # compile and call a function. This prevents theano errors. return # Need to make sure there's only one update per variable for the # case where we've got the same module instance at multiple places # within the graph. # Also warn about it because it's not obvious whether just dropping # one of them is the right thing to do in general? todo = set(upd[0] for upd in stat_updates) if len(todo) < len(stat_updates): uniq_updates = [] for upd in stat_updates: if upd[0] in todo: uniq_updates.append(upd) todo.remove(upd[0]) else: print("WARNING: Dropped the following stat-update because that variable got multiple updates: {}".format(upd[0])) stat_updates = uniq_updates self._fn_accum_stats[self.training_mode] = df.th.function( inputs=aslist(symb_in), updates=stat_updates ) self._fn_accum_stats[self.training_mode](*aslist(data_in))
def accumulate_statistics(self, data_in): if self.training_mode not in self._fn_accum_stats: symb_in = make_tensor_or_tensors(data_in, 'X') # Call forward once so it can compute some variables it'll actually # use in the stat updates collection. self.symb_forward(symb_in) stat_updates = self.get_stat_updates() if not stat_updates: # If there's no layer collecting statistics, we don't need to # compile and call a function. This prevents theano errors. return # Need to make sure there's only one update per variable for the # case where we've got the same module instance at multiple places # within the graph. # Also warn about it because it's not obvious whether just dropping # one of them is the right thing to do in general? todo = set(upd[0] for upd in stat_updates) if len(todo) < len(stat_updates): uniq_updates = [] for upd in stat_updates: if upd[0] in todo: uniq_updates.append(upd) todo.remove(upd[0]) else: print( "WARNING: Dropped the following stat-update because that variable got multiple updates: {}" .format(upd[0])) stat_updates = uniq_updates self._fn_accum_stats[self.training_mode] = df.th.function( inputs=aslist(symb_in), updates=stat_updates) self._fn_accum_stats[self.training_mode](*aslist(data_in))