def test_cumsumOp(self):
x = T.tensor3('x')
a = np.random.random((3, 5, 2)).astype(config.floatX)
f = theano.function([x], cumsum(x))
assert np.allclose(np.cumsum(a), f(a)) # Test axis=None
for axis in range(len(a.shape)):
f = theano.function([x], cumsum(x, axis=axis))
assert np.allclose(np.cumsum(a, axis=axis), f(a))
def gradient(self, observed, at_risk):
prediction = self.output
risk = T.exp(prediction)
product = self.input * (risk * T.ones((1, self.input.shape[0])))
numerator = Te.cumsum(product[::-1])[::-1][at_risk]
denominator = Te.cumsum(risk[::-1])[::-1][at_risk] * T.ones((1, self.input.shape[0]))
numerator = numerator.flatten()
denominator = denominator.flatten()
gradient = T.dot(observed, self.input - (numerator / denominator))
return gradient
def test_cumsumOp(self):
x = T.tensor3('x')
a = np.random.random((3, 5, 2)).astype(config.floatX)
# Test axis out of bounds
self.assertRaises(ValueError, cumsum, x, axis=3)
self.assertRaises(ValueError, cumsum, x, axis=-4)
f = theano.function([x], cumsum(x))
assert np.allclose(np.cumsum(a), f(a)) # Test axis=None
for axis in range(-len(a.shape), len(a.shape)):
f = theano.function([x], cumsum(x, axis=axis))
assert np.allclose(np.cumsum(a, axis=axis), f(a))
def cost(self, observed, at_risk):
prediction = self.output
exp = T.exp(prediction)[::-1]
partial_sum = Te.cumsum(exp)[::-1] + 1 # get the reversed partial cumulative sum
log_at_risk = T.log(partial_sum[at_risk])
diff = prediction - log_at_risk
cost = T.sum(T.dot(observed, diff))
return cost
def test_infer_shape(self):
x = T.tensor3("x")
a = np.random.random((3, 5, 2)).astype(config.floatX)
# Test axis=None
self._compile_and_check([x], [self.op(x)], [a], self.op_class)
for axis in range(-len(a.shape), len(a.shape)):
self._compile_and_check([x], [cumsum(x, axis=axis)], [a], self.op_class)
def test_infer_shape(self):
x = T.tensor3('x')
a = np.random.random((3, 5, 2)).astype(theano.config.floatX)
for axis in range(-len(a.shape), len(a.shape)):
self._compile_and_check([x],
[cumsum(x, axis=axis)],
[a],
self.op_class)
def test_Strides3D(self):
x = T.ftensor3('x')
for axis in [0, 1, 2, None]:
a = np.random.random((42, 30, 25)).astype("float32")
cumsum_function = theano.function([x], cumsum(x, axis=axis), mode=self.mode)
slicings = [slice(None, None, None), # Normal strides
slice(None, None, 2), # Stepped strides
slice(None, None, -1), # Negative strides
]
# Cartesian product of all slicings to test.
for slicing in itertools.product(slicings, repeat=x.ndim):
f = theano.function([x], cumsum(x[slicing], axis=axis), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
assert np.allclose(np.cumsum(a[slicing], axis=axis), f(a))
assert np.allclose(np.cumsum(a[slicing], axis=axis), cumsum_function(a[slicing]))
def test_Strides1D(self):
x = T.fvector('x')
for axis in [0, None, -1]:
a = np.random.random((42,)).astype("float32")
cumsum_function = theano.function([x], cumsum(x, axis=axis),
mode=self.mode)
slicings = [slice(None, None, None), # Normal strides
slice(None, None, 2), # Stepped strides
slice(None, None, -1), # Negative strides
]
# Cartesian product of all slicings to test.
for slicing in itertools.product(slicings, repeat=x.ndim):
f = theano.function([x], cumsum(x[slicing], axis=axis),
mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
utt.assert_allclose(np.cumsum(a[slicing], axis=axis), f(a))
utt.assert_allclose(np.cumsum(a[slicing], axis=axis),
cumsum_function(a[slicing]))
def test_cum_op(self):
x = T.tensor3('x')
a = np.random.random((3, 5, 2)).astype(config.floatX)
# Test axis out of bounds
self.assertRaises(ValueError, cumsum, x, axis=3)
self.assertRaises(ValueError, cumsum, x, axis=-4)
self.assertRaises(ValueError, cumprod, x, axis=3)
self.assertRaises(ValueError, cumprod, x, axis=-4)
f = theano.function([x], [cumsum(x), cumprod(x)])
s, p = f(a)
assert np.allclose(np.cumsum(a), s) # Test axis=None
assert np.allclose(np.cumprod(a), p) # Test axis=None
for axis in range(-len(a.shape), len(a.shape)):
f = theano.function([x],
[cumsum(x, axis=axis),
cumprod(x, axis=axis)])
s, p = f(a)
assert np.allclose(np.cumsum(a, axis=axis), s)
assert np.allclose(np.cumprod(a, axis=axis), p)
def my_keras_cumsum(tensor, axis=0):
"""
Keras doesn't have a cumsum operation yet, but it seems to be nearly there - see this PR:
https://github.com/fchollet/keras/pull/3791.
"""
# Putting the imports here so that tests and things don't have to run them, unless
# they're using this encoder with the particular backend.
if K.backend() == "tensorflow":
import tensorflow as tf
return tf.cumsum(tensor, axis=axis)
else:
import theano.tensor.extra_ops as T
return T.cumsum(tensor, axis=axis)
def test_GpuCumsum3D(self):
block_max_size = self.max_threads_dim0 * 2
x = T.ftensor3('x')
for shape_axis, axis in zip([0, 1, 2, 0, 2, 1, 0], [0, 1, 2, None, -1, -2, -3]):
f = theano.function([x], cumsum(x, axis=axis), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
# Extensive testing for the first 1025 sizes
a_shape = [5, 5, 5]
a_shape[shape_axis] = 1025
a = np.random.rand(*a_shape).astype("float32")
slices = [slice(None), slice(None), slice(None)]
for i in xrange(a.shape[shape_axis]):
slices[shape_axis] = slice(i)
fa = f(a[slices])
npa = np.cumsum(a[slices], axis=axis)
utt.assert_allclose(npa, fa)
# Use multiple GPU threadblocks (along accumulation axis)
a_shape = [2, 2, 2]
a_shape[shape_axis] = block_max_size + 2
a = np.random.random(a_shape).astype("float32")
utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
# Use multiple GPU gridblocks (not along accumulation axis)
a_shape = [5, 5, 5]
a_shape[(shape_axis + 1) % 3] = self.max_grid_size1 + 1
a = np.random.random(a_shape).astype("float32")
if axis is None:
# Avoid floating point error
a = np.sign(a - 0.5).astype("float32")
utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
a_shape = [5, 5, 5]
a_shape[(shape_axis + 2) % 3] = self.max_grid_size1 + 1
a = np.random.random(a_shape).astype("float32")
if axis is None:
# Avoid floating point error
a = np.sign(a - 0.5).astype("float32")
utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
# Use recursive cumsum (along accumulation axis)
a_shape = [3, 3, 3]
a_shape[shape_axis] = block_max_size * (
block_max_size + 1) + 2
a = np.random.random(a_shape).astype("float32")
a = np.sign(a - 0.5).astype(
"float32") # Avoid floating point error
utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
def test_Strides1D(self):
x = T.fvector('x')
# Stepped strides
f = theano.function([x], cumsum(x[::2]), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
a = np.random.randint(10, size=(42,)).astype("float32")
assert np.allclose(np.cumsum(a[::2]), f(a))
# Alternative stepped strides
f = theano.function([x], cumsum(x), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
a = np.random.randint(10, size=(42,)).astype("float32")
assert np.allclose(np.cumsum(a[::2]), f(a[::2]))
# Negative strides
f = theano.function([x], cumsum(x[::-1]), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
a = np.random.randint(10, size=(42,)).astype("float32")
assert np.allclose(np.cumsum(a[::-1]), f(a))
def test_GpuCumsum3D(self):
block_max_size = self.max_threads_dim0 * 2
x = T.ftensor3('x')
for shape_axis, axis in zip([0, 1, 2, 0, 2, 1, 0], [0, 1, 2, None, -1, -2, -3]):
f = theano.function([x], cumsum(x, axis=axis), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
# Extensive testing for the first 1025 sizes
a_shape = [5, 5, 5]
a_shape[shape_axis] = 1025
a = np.random.rand(*a_shape).astype("float32")
slices = [slice(None), slice(None), slice(None)]
for i in xrange(a.shape[shape_axis]):
slices[shape_axis] = slice(i)
fa = f(a[slices])
npa = np.cumsum(a[slices], axis=axis)
utt.assert_allclose(npa, fa)
# Use multiple GPU threadblocks (along accumulation axis)
a_shape = [2, 2, 2]
a_shape[shape_axis] = block_max_size + 2
a = np.random.random(a_shape).astype("float32")
utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
# Use multiple GPU gridblocks (not along accumulation axis)
a_shape = [5, 5, 5]
a_shape[(shape_axis + 1) % 3] = self.max_grid_size1 + 1
a = np.random.random(a_shape).astype("float32")
if axis is None:
# Avoid floating point error
a = np.sign(a - 0.5).astype("float32")
utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
a_shape = [5, 5, 5]
a_shape[(shape_axis + 2) % 3] = self.max_grid_size1 + 1
a = np.random.random(a_shape).astype("float32")
if axis is None:
# Avoid floating point error
a = np.sign(a - 0.5).astype("float32")
utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
# Use recursive cumsum (along accumulation axis)
a_shape = [3, 3, 3]
a_shape[shape_axis] = block_max_size * (block_max_size + 1) + 2
a = np.random.random(a_shape).astype("float32")
a = np.sign(a - 0.5).astype("float32") # Avoid floating point error
utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
def test_infer_shape(self):
# GpuCumSum is only defined for float32 for now, so we skip it
# in the unsupported cases
gpucumsum_supported_dtypes = ('float32',)
if theano.config.floatX not in gpucumsum_supported_dtypes:
raise SkipTest('GpuCumSum not implemented for dtype %s'
% theano.config.floatX)
x = T.tensor3('x')
a = np.random.random((3, 5, 2)).astype(theano.config.floatX)
for axis in range(-len(a.shape), len(a.shape)):
self._compile_and_check([x],
[cumsum(x, axis=axis)],
[a],
self.op_class)
def test_infer_shape(self):
x = T.tensor3('x')
a = np.random.random((3, 5, 2)).astype(config.floatX)
# Test axis=None
self._compile_and_check([x],
[self.op(x)],
[a],
self.op_class)
for axis in range(-len(a.shape), len(a.shape)):
self._compile_and_check([x],
[cumsum(x, axis=axis)],
[a],
self.op_class)
def test_GpuCumsum2D(self):
block_max_size = self.max_threads_dim0 * 2
x = T.fmatrix('x')
for shape_axis, axis in zip([0, 1, 0], [0, 1, None]):
f = theano.function([x], cumsum(x, axis=axis), mode=self.mode)
assert [
n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)
]
# Extensive testing for the first 1025 sizes
a_shape = [5, 5]
a_shape[shape_axis] = 1025
a = np.random.random(a_shape).astype("float32")
slices = [slice(None), slice(None)]
for i in xrange(a.shape[shape_axis]):
slices[shape_axis] = slice(i)
fa = f(a[slices])
npa = np.cumsum(a[slices], axis=axis)
assert np.allclose(npa, fa)
# Use multiple GPU threadblocks
a_shape = [5, 5]
a_shape[shape_axis] = block_max_size + 2
a = np.random.random(a_shape).astype("float32")
assert np.allclose(np.cumsum(a, axis=axis), f(a))
# Use multiple GPU gridblocks
a_shape = [5, 5]
a_shape[1 - shape_axis] = self.max_grid_size1 + 1
a = np.random.random(a_shape).astype("float32")
assert np.allclose(np.cumsum(a, axis=axis), f(a))
# Use recursive cumsum
a_shape = [3, 3]
a_shape[shape_axis] = block_max_size * (block_max_size + 1) + 2
a = np.random.random(a_shape).astype("float32")
a = np.sign(a - 0.5).astype(
"float32") # Avoid floating point error
assert np.allclose(np.cumsum(a, axis=axis), f(a))
def test_GpuCumsum1D(self):
block_max_size = self.max_threads_dim0 * 2
x = T.fvector('x')
f = theano.function([x], cumsum(x), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
# Extensive testing for the first 1025 sizes
a = np.random.random(1025).astype("float32")
for i in xrange(a.shape[0]):
utt.assert_allclose(np.cumsum(a[:i]), f(a[:i]))
# Use multiple GPU threadblocks
a = np.random.random((block_max_size + 2, )).astype("float32")
utt.assert_allclose(np.cumsum(a), f(a))
# Use recursive cumsum
a = np.ones((block_max_size * (block_max_size + 1) + 2,),
dtype="float32")
utt.assert_allclose(np.cumsum(a), f(a))
def test_GpuCumsum1D(self):
block_max_size = self.max_threads_dim0 * 2
x = T.fvector('x')
f = theano.function([x], cumsum(x), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
# Extensive testing for the first 1025 sizes
a = np.random.random(1025).astype("float32")
for i in xrange(a.shape[0]):
utt.assert_allclose(np.cumsum(a[:i]), f(a[:i]))
# Use multiple GPU threadblocks
a = np.random.random((block_max_size + 2, )).astype("float32")
utt.assert_allclose(np.cumsum(a), f(a))
# Use recursive cumsum
a = np.ones((block_max_size * (block_max_size + 1) + 2,),
dtype="float32")
utt.assert_allclose(np.cumsum(a), f(a))
def cost(self, observed, at_risk): """Calculates the cox negative log likelihood. Args: observed: 1D array. Event status; 0 means censored. at_risk: 1D array. Element i of this array indicates the index of the first patient in the at risk group of patient i, when patients are sorted by increasing time to event. Returns: Objective function to be maximized. """ prediction = self.output # Subtracts maximum to facilitate computation. factorizedPred = prediction - prediction.max() exp = T.exp(factorizedPred)[::-1] # Calculates the reversed partial cumulative sum. partial_sum = Te.cumsum(exp)[::-1] + 1 # Adds the subtracted maximum back. log_at_risk = T.log(partial_sum[at_risk]) + prediction.max() diff = prediction - log_at_risk cost = T.sum(T.dot(observed, diff)) return cost
def test_GpuCumsum2D(self):
block_max_size = self.max_threads_dim0 * 2
x = T.fmatrix('x')
for shape_axis, axis in zip([0, 1, 0], [0, 1, None]):
f = theano.function([x], cumsum(x, axis=axis), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
# Extensive testing for the first 1025 sizes
a_shape = [5, 5]
a_shape[shape_axis] = 1025
a = np.random.random(a_shape).astype("float32")
slices = [slice(None), slice(None)]
for i in xrange(a.shape[shape_axis]):
slices[shape_axis] = slice(i)
fa = f(a[slices])
npa = np.cumsum(a[slices], axis=axis)
utt.assert_allclose(npa, fa)
# Use multiple GPU threadblocks
a_shape = [5, 5]
a_shape[shape_axis] = block_max_size+2
a = np.random.random(a_shape).astype("float32")
utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
# Use multiple GPU gridblocks
a_shape = [5, 5]
a_shape[1-shape_axis] = self.max_grid_size1+1
a = np.random.random(a_shape).astype("float32")
utt.assert_allclose(np.cumsum(a, axis=axis), f(a), rtol=5e-5)
# Use recursive cumsum
a_shape = [3, 3]
a_shape[shape_axis] = block_max_size*(block_max_size+1)+2
a = np.random.random(a_shape).astype("float32")
a = np.sign(a-0.5).astype("float32") # Avoid floating point error
utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
def _collect_samples(self, y):
"""
This function collect N samples of size T using the current policy.
:param y:
:return: locations (n_batch, N, T, 2), probabilities (n_batch, N, T, n_classes),
rewards (n_batch, N, T, ) and returns (n_batch, N, T, )
"""
means = []
locs = []
probs = []
returns = []
preds = []
# Reshape target labels to match the classification outputs along each path of length T
y_rep = T.stack([
T.fill(T.zeros((self.policy.n_steps)), y[b])
for b in xrange(self.policy.n_batch)
],
axis=0)
for _ in xrange(self.policy.N):
loc_means_t, locs_t, _, x_ts, p_ts = self.policy.step_forward()
locs.append(locs_t)
means.append(loc_means_t)
probs.append(p_ts)
pred = np.argmax(p_ts, axis=2)
preds.append(pred)
rewards = self._acc_score(pred, y_rep)
returns.append(cumsum(rewards, axis=1))
locs = T.stack(locs).dimshuffle(1, 0, *range(2, T.stack(locs).ndim))
means = T.stack(means).dimshuffle(1, 0, *range(2, T.stack(means).ndim))
preds = T.stack(preds).dimshuffle(1, 0, *range(2, T.stack(preds).ndim))
returns = T.stack(returns).dimshuffle(1, 0,
*range(2,
T.stack(returns).ndim))
return locs, means, preds, returns
def test_Strides2D(self):
x = T.fmatrix('x')
for shape_axis, axis in zip([0, 1, 0], [0, 1, None]):
a = np.random.random((42, 30)).astype("float32")
# Stepped strides along axis=0
f = theano.function([x], cumsum(x[::2], axis=axis), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
assert np.allclose(np.cumsum(a[::2], axis=axis), f(a))
# Stepped strides along axis=1
f = theano.function([x], cumsum(x[:, ::2], axis=axis), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
assert np.allclose(np.cumsum(a[:, ::2], axis=axis), f(a))
# Alternative stepped strides along axis=0
f = theano.function([x], cumsum(x), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
assert np.allclose(np.cumsum(a[::2]), f(a[::2]))
# Alternative stepped strides along axis=1
f = theano.function([x], cumsum(x), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
assert np.allclose(np.cumsum(a[:, ::2]), f(a[:, ::2]))
# Negative strides along axis=0
f = theano.function([x], cumsum(x[::-1], axis=axis), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
assert np.allclose(np.cumsum(a[::-1], axis=axis), f(a))
# Negative strides along axis=1
f = theano.function([x], cumsum(x[:, ::-1], axis=axis), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, GpuCumsum)]
assert np.allclose(np.cumsum(a[:, ::-1], axis=axis), f(a))
def test_GpuCumsum4D(self):
# Should not use the GPU version.
x = T.ftensor4('x')
f = theano.function([x], cumsum(x, axis=1), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, CumsumOp)]
)
layer2 = ll.LSTMLayer(
input = layer_embed.output,
n_in = layer_embed.n_out,
n_out = 150,
backwards=True
)
#gets only the output vectors whose indices are equal to the end of token
end_indices = T.nonzero(T.eq(T.argmax(x, axis=1),input_dim-1))
#gets only the output vectors whose indices are equal to the start of token
start_indices = T.nonzero(T.eq(T.argmax(x,axis=1),input_dim-2))
hl = T.concatenate((layer1.output, layer2.output[::-1]),axis=1)
hc = extra.cumsum(hl, axis=0)
hsub = hc[end_indices] - hc[start_indices]
diff_indices = T.as_tensor_variable(end_indices) - T.as_tensor_variable(start_indices)
diff_shuf = diff_indices.flatten().dimshuffle(0, 'x')
h = hsub / diff_shuf
h_size = (layer1.n_out + layer2.n_out)
#relationship between near words
layer_c = nl.NNLayer(
input = h,
n_in = h_size,
n_out = 150,
activation = T.tanh)
def kl(Y, Y_hat, cost_mask=None,
batch_vec=True,
cost_matrix=False,
sum_tru_time=False,
normalize_by_outsize=True):
"""
Warning: This function expects a sigmoid nonlinearity in the
output layer. Returns a batch (vector) of mean across units of
KL divergence for each example,
KL(P || Q) where P is defined by Y and Q is defined by Y_hat:
p log p - p log q + (1-p) log (1-p) - (1-p) log (1-q)
For binary p, some terms drop out:
- p log q - (1-p) log (1-q)
- p log sigmoid(z) - (1-p) log sigmoid(-z)
p softplus(-z) + (1-p) softplus(z)
Parameters
----------
Y : Variable
targets for the sigmoid outputs. Currently Y must be purely binary.
If it's not, you'll still get the right gradient, but the
value in the monitoring channel will be wrong.
Y_hat : Variable
predictions made by the sigmoid layer. Y_hat must be generated by
fprop, i.e., it must be a symbolic sigmoid.
-------
ave : Variable
average kl divergence between Y and Y_hat.
"""
assert hasattr(Y_hat, 'owner')
owner = Y_hat.owner
assert owner is not None
op = owner.op
if not hasattr(op, 'scalar_op'):
raise ValueError("Expected Y_hat to be generated by an Elemwise "
"op, got "+str(op)+" of type "+str(type(op)))
assert isinstance(op.scalar_op, TT.nnet.sigm.ScalarSigmoid)
z, = owner.inputs
z = z.reshape(Y.shape)
term_1 = Y * TT.nnet.softplus(-z)
term_2 = (1 - Y) * TT.nnet.softplus(z)
total = term_1 + term_2
if cost_mask is not None:
if cost_mask.ndim != total.ndim:
cost_mask = cost_mask.dimshuffle(0, 1, 'x')
total = cost_mask * total
if not sum_tru_time:
if normalize_by_outsize:
if cost_matrix:
ave = total.sum(-1) / TT.cast((total.shape[2] - 2), "float32")
else:
if batch_vec:
ave = total.sum(0).sum(1) / TT.cast((total.shape[2] - 2), "float32")
else:
ave = total.sum() / (total.shape[1] * (total.shape[2] - 2))
else:
if cost_matrix:
ave = total.sum(-1)
else:
if batch_vec:
ave = total.sum(0).sum(1)
else:
ave = total.sum() / TT.cast(total.shape[1], "float32")
else:
assert not cost_matrix
if normalize_by_outsize:
if batch_vec:
ave = cumsum(total.sum(-1) / TT.cast((total.shape[2] - 2), "float32"),
axis=0)[::-1]
else:
ave = cumsum(total.sum((1, 2)) / (total.shape[1] * (total.shape[2] - 2)),
axis=0)[::-1]
else:
if batch_vec:
ave = cumsum(total.sum(-1), axis=0)[::-1]
else:
ave = cumsum(total.sum((1, 2)) / TT.cast(total.shape[1], "float32"), axis=0)[::-1]
return ave
def test_GpuCumsum4D(self):
# Should not use the GPU version.
x = T.ftensor4('x')
f = theano.function([x], cumsum(x, axis=1), mode=self.mode)
assert [n for n in f.maker.fgraph.toposort()
if isinstance(n.op, CumsumOp)]
def kl(Y,
Y_hat,
cost_mask=None,
batch_vec=True,
cost_matrix=False,
sum_tru_time=False,
normalize_by_outsize=True):
"""
Warning: This function expects a sigmoid nonlinearity in the
output layer. Returns a batch (vector) of mean across units of
KL divergence for each example,
KL(P || Q) where P is defined by Y and Q is defined by Y_hat:
p log p - p log q + (1-p) log (1-p) - (1-p) log (1-q)
For binary p, some terms drop out:
- p log q - (1-p) log (1-q)
- p log sigmoid(z) - (1-p) log sigmoid(-z)
p softplus(-z) + (1-p) softplus(z)
Parameters
----------
Y : Variable
targets for the sigmoid outputs. Currently Y must be purely binary.
If it's not, you'll still get the right gradient, but the
value in the monitoring channel will be wrong.
Y_hat : Variable
predictions made by the sigmoid layer. Y_hat must be generated by
fprop, i.e., it must be a symbolic sigmoid.
-------
ave : Variable
average kl divergence between Y and Y_hat.
"""
assert hasattr(Y_hat, 'owner')
owner = Y_hat.owner
assert owner is not None
op = owner.op
if not hasattr(op, 'scalar_op'):
raise ValueError("Expected Y_hat to be generated by an Elemwise "
"op, got " + str(op) + " of type " + str(type(op)))
assert isinstance(op.scalar_op, TT.nnet.sigm.ScalarSigmoid)
z, = owner.inputs
z = z.reshape(Y.shape)
term_1 = Y * TT.nnet.softplus(-z)
term_2 = (1 - Y) * TT.nnet.softplus(z)
total = term_1 + term_2
if cost_mask is not None:
if cost_mask.ndim != total.ndim:
cost_mask = cost_mask.dimshuffle(0, 1, 'x')
total = cost_mask * total
if not sum_tru_time:
if normalize_by_outsize:
if cost_matrix:
ave = total.sum(-1) / TT.cast((total.shape[2] - 2), "float32")
else:
if batch_vec:
ave = total.sum(0).sum(1) / TT.cast(
(total.shape[2] - 2), "float32")
else:
ave = total.sum() / (total.shape[1] * (total.shape[2] - 2))
else:
if cost_matrix:
ave = total.sum(-1)
else:
if batch_vec:
ave = total.sum(0).sum(1)
else:
ave = total.sum() / TT.cast(total.shape[1], "float32")
else:
assert not cost_matrix
if normalize_by_outsize:
if batch_vec:
ave = cumsum(total.sum(-1) / TT.cast(
(total.shape[2] - 2), "float32"),
axis=0)[::-1]
else:
ave = cumsum(total.sum(
(1, 2)) / (total.shape[1] * (total.shape[2] - 2)),
axis=0)[::-1]
else:
if batch_vec:
ave = cumsum(total.sum(-1), axis=0)[::-1]
else:
ave = cumsum(total.sum(
(1, 2)) / TT.cast(total.shape[1], "float32"),
axis=0)[::-1]
return ave
