def forward(self, data):
if self.training_mode not in self._fn_forward:
symb_in = make_tensor_or_tensors(data, 'X')
symb_out = self.symb_forward(symb_in)
self._fn_forward[self.training_mode] = df.th.function(
inputs=aslist(symb_in), outputs=symb_out)
return self._fn_forward[self.training_mode](*aslist(data))
def forward(self, data):
if self.training_mode not in self._fn_forward:
symb_in = make_tensor_or_tensors(data, 'X')
symb_out = self.symb_forward(symb_in)
self._fn_forward[self.training_mode] = df.th.function(
inputs=aslist(symb_in),
outputs=symb_out
)
return self._fn_forward[self.training_mode](*aslist(data))
def accumulate_gradients(self, data_in, data_tgt, loss):
if self.training_mode not in self._fn_accum_grads:
symb_in = make_tensor_or_tensors(data_in, 'X')
symb_tgt = make_tensor_or_tensors(data_tgt, 'T')
symb_out = self.symb_forward(symb_in)
symb_err = loss.symb_forward(symb_out, symb_tgt)
params, grads = self.unique_parameters()
symb_grads = df.th.grad(cost=symb_err, wrt=params)
grads_updates = [(grad, grad + symb_grad) for grad, symb_grad in zip(grads, symb_grads)]
self._fn_accum_grads[self.training_mode] = df.th.function(
inputs=aslist(symb_in) + aslist(symb_tgt),
outputs=symb_err,
updates=grads_updates
)
args = aslist(data_in) + aslist(data_tgt)
return self._fn_accum_grads[self.training_mode](*args)
def accumulate_gradients(self, data_in, data_tgt, loss):
if self.training_mode not in self._fn_accum_grads:
symb_in = make_tensor_or_tensors(data_in, 'X')
symb_tgt = make_tensor_or_tensors(data_tgt, 'T')
symb_out = self.symb_forward(symb_in)
symb_err = loss.symb_forward(symb_out, symb_tgt)
params, grads = self.unique_parameters()
symb_grads = df.th.grad(cost=symb_err, wrt=params)
grads_updates = [(grad, grad + symb_grad)
for grad, symb_grad in zip(grads, symb_grads)]
self._fn_accum_grads[self.training_mode] = df.th.function(
inputs=aslist(symb_in) + aslist(symb_tgt),
outputs=symb_err,
updates=grads_updates)
args = aslist(data_in) + aslist(data_tgt)
return self._fn_accum_grads[self.training_mode](*args)
def accumulate_statistics(self, data_in):
if self.training_mode not in self._fn_accum_stats:
symb_in = make_tensor_or_tensors(data_in, 'X')
# Call forward once so it can compute some variables it'll actually
# use in the stat updates collection.
self.symb_forward(symb_in)
stat_updates = self.get_stat_updates()
if not stat_updates:
# If there's no layer collecting statistics, we don't need to
# compile and call a function. This prevents theano errors.
return
# Need to make sure there's only one update per variable for the
# case where we've got the same module instance at multiple places
# within the graph.
# Also warn about it because it's not obvious whether just dropping
# one of them is the right thing to do in general?
todo = set(upd[0] for upd in stat_updates)
if len(todo) < len(stat_updates):
uniq_updates = []
for upd in stat_updates:
if upd[0] in todo:
uniq_updates.append(upd)
todo.remove(upd[0])
else:
print("WARNING: Dropped the following stat-update because that variable got multiple updates: {}".format(upd[0]))
stat_updates = uniq_updates
self._fn_accum_stats[self.training_mode] = df.th.function(
inputs=aslist(symb_in),
updates=stat_updates
)
self._fn_accum_stats[self.training_mode](*aslist(data_in))
def accumulate_statistics(self, data_in):
if self.training_mode not in self._fn_accum_stats:
symb_in = make_tensor_or_tensors(data_in, 'X')
# Call forward once so it can compute some variables it'll actually
# use in the stat updates collection.
self.symb_forward(symb_in)
stat_updates = self.get_stat_updates()
if not stat_updates:
# If there's no layer collecting statistics, we don't need to
# compile and call a function. This prevents theano errors.
return
# Need to make sure there's only one update per variable for the
# case where we've got the same module instance at multiple places
# within the graph.
# Also warn about it because it's not obvious whether just dropping
# one of them is the right thing to do in general?
todo = set(upd[0] for upd in stat_updates)
if len(todo) < len(stat_updates):
uniq_updates = []
for upd in stat_updates:
if upd[0] in todo:
uniq_updates.append(upd)
todo.remove(upd[0])
else:
print(
"WARNING: Dropped the following stat-update because that variable got multiple updates: {}"
.format(upd[0]))
stat_updates = uniq_updates
self._fn_accum_stats[self.training_mode] = df.th.function(
inputs=aslist(symb_in), updates=stat_updates)
self._fn_accum_stats[self.training_mode](*aslist(data_in))
