def forward(self, num_input, num_target, with_penalties=True, per_sample=False):
# NOTE: using the GPU for such trivial computations as most costs
# is actually somewhat slower (e.g. for RMSE: GPU 1.2ms vs. CPU 0.2ms).
# So ideally, we'd like to compile a CPU-version here, but I don't know how!
if (with_penalties, per_sample) not in self._fn_forward:
symb_in = tensors_for_ndarrays(num_input, 'Y')
symb_tgt = tensors_for_ndarrays(num_target, 'T')
symb_out = self(symb_in, symb_tgt, with_penalties)
self._fn_forward[with_penalties, per_sample] = df.th.function(
inputs=flatten(symb_in) + flatten(symb_tgt),
outputs=symb_out if not per_sample else self._per_sample_cost
)
return self._fn_forward[with_penalties, per_sample](*(flatten(num_input) + flatten(num_target)))
def accumulate_gradients(self, data_in, data_tgt, crit):
if (self._mode, id(crit)) not in self._fn_accum_grads:
symb_in = tensors_for_ndarrays(data_in, 'X')
symb_tgt = tensors_for_ndarrays(data_tgt, 'T')
symb_out = self(symb_in)
symb_cost = crit(symb_out, symb_tgt)
extra_out = self.get_extra_outputs() + crit.get_extra_outputs()
params = self.parameters(learnable_only=True)
symb_grads = df.th.grad(cost=symb_cost, wrt=[p.param for p in params])
grads_updates = [(p.grad, p.grad + symb_grad) for p, symb_grad in zip(params, symb_grads)]
fn = self._fn_accum_grads[self._mode, id(crit)] = df.th.function(
inputs=flatten(symb_in) + flatten(symb_tgt),
outputs=flatten(symb_cost) + flatten(extra_out),
updates=grads_updates
)
fn._df2_extra = extra_out
fn = self._fn_accum_grads[self._mode, id(crit)]
args = flatten(data_in) + flatten(data_tgt)
outs = fn(*args)
return self._collect_extra_outputs(fn, outs)
def forward(self, data):
if self._mode not in self._fn_forward:
symb_in = tensors_for_ndarrays(data, 'X')
symb_out = self(symb_in)
extra_out = self.get_extra_outputs()
fn = self._fn_forward[self._mode] = df.th.function(
inputs=flatten(symb_in),
outputs=flatten(symb_out) + flatten(extra_out)
)
fn._df2_extra = extra_out
fn = self._fn_forward[self._mode]
outs = fn(*flatten(data))
return self._collect_extra_outputs(fn, outs)
def accumulate_statistics(self, data_in):
if self._mode not in self._fn_accum_stats:
symb_in = tensors_for_ndarrays(data_in, 'X')
# Call forward once so it can compute some variables it'll actually
# use in the stat updates collection.
self(symb_in)
stat_updates = self.get_stat_updates()
if not stat_updates:
# If there's no layer collecting statistics, we don't need to
# compile and call a function. This prevents theano errors.
return
# Need to make sure there's only one update per variable for the
# case where we've got the same module instance at multiple places
# within the graph.
# Also warn about it because it's not obvious whether just dropping
# one of them is the right thing to do in general?
todo = set(upd[0] for upd in stat_updates)
if len(todo) < len(stat_updates):
uniq_updates = []
for upd in stat_updates:
if upd[0] in todo:
uniq_updates.append(upd)
todo.remove(upd[0])
else:
print("WARNING: Dropped the following stat-update because that variable got multiple updates: {}".format(upd[0]))
stat_updates = uniq_updates
self._fn_accum_stats[self._mode] = df.th.function(
inputs=flatten(symb_in),
updates=stat_updates
)
self._fn_accum_stats[self._mode](*flatten(data_in))
