def test_cumsumOp(self): x = T.tensor3('x') a = np.random.random((3, 5, 2)).astype(config.floatX) f = theano.function([x], cumsum(x)) assert np.allclose(np.cumsum(a), f(a)) # Test axis=None for axis in range(len(a.shape)): f = theano.function([x], cumsum(x, axis=axis)) assert np.allclose(np.cumsum(a, axis=axis), f(a))
def gradient(self, observed, at_risk): prediction = self.output risk = T.exp(prediction) product = self.input * (risk * T.ones((1, self.input.shape[0]))) numerator = Te.cumsum(product[::-1])[::-1][at_risk] denominator = Te.cumsum(risk[::-1])[::-1][at_risk] * T.ones((1, self.input.shape[0])) numerator = numerator.flatten() denominator = denominator.flatten() gradient = T.dot(observed, self.input - (numerator / denominator)) return gradient
def test_cumsumOp(self): x = T.tensor3('x') a = np.random.random((3, 5, 2)).astype(config.floatX) # Test axis out of bounds self.assertRaises(ValueError, cumsum, x, axis=3) self.assertRaises(ValueError, cumsum, x, axis=-4) f = theano.function([x], cumsum(x)) assert np.allclose(np.cumsum(a), f(a)) # Test axis=None for axis in range(-len(a.shape), len(a.shape)): f = theano.function([x], cumsum(x, axis=axis)) assert np.allclose(np.cumsum(a, axis=axis), f(a))
def cost(self, observed, at_risk): prediction = self.output exp = T.exp(prediction)[::-1] partial_sum = Te.cumsum(exp)[::-1] + 1 # get the reversed partial cumulative sum log_at_risk = T.log(partial_sum[at_risk]) diff = prediction - log_at_risk cost = T.sum(T.dot(observed, diff)) return cost
def test_infer_shape(self): x = T.tensor3("x") a = np.random.random((3, 5, 2)).astype(config.floatX) # Test axis=None self._compile_and_check([x], [self.op(x)], [a], self.op_class) for axis in range(-len(a.shape), len(a.shape)): self._compile_and_check([x], [cumsum(x, axis=axis)], [a], self.op_class)
def test_infer_shape(self): x = T.tensor3('x') a = np.random.random((3, 5, 2)).astype(theano.config.floatX) for axis in range(-len(a.shape), len(a.shape)): self._compile_and_check([x], [cumsum(x, axis=axis)], [a], self.op_class)
def test_Strides3D(self): x = T.ftensor3('x') for axis in [0, 1, 2, None]: a = np.random.random((42, 30, 25)).astype("float32") cumsum_function = theano.function([x], cumsum(x, axis=axis), mode=self.mode) slicings = [slice(None, None, None), # Normal strides slice(None, None, 2), # Stepped strides slice(None, None, -1), # Negative strides ] # Cartesian product of all slicings to test. for slicing in itertools.product(slicings, repeat=x.ndim): f = theano.function([x], cumsum(x[slicing], axis=axis), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] assert np.allclose(np.cumsum(a[slicing], axis=axis), f(a)) assert np.allclose(np.cumsum(a[slicing], axis=axis), cumsum_function(a[slicing]))
def test_Strides1D(self): x = T.fvector('x') for axis in [0, None, -1]: a = np.random.random((42,)).astype("float32") cumsum_function = theano.function([x], cumsum(x, axis=axis), mode=self.mode) slicings = [slice(None, None, None), # Normal strides slice(None, None, 2), # Stepped strides slice(None, None, -1), # Negative strides ] # Cartesian product of all slicings to test. for slicing in itertools.product(slicings, repeat=x.ndim): f = theano.function([x], cumsum(x[slicing], axis=axis), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] utt.assert_allclose(np.cumsum(a[slicing], axis=axis), f(a)) utt.assert_allclose(np.cumsum(a[slicing], axis=axis), cumsum_function(a[slicing]))
def test_cum_op(self): x = T.tensor3('x') a = np.random.random((3, 5, 2)).astype(config.floatX) # Test axis out of bounds self.assertRaises(ValueError, cumsum, x, axis=3) self.assertRaises(ValueError, cumsum, x, axis=-4) self.assertRaises(ValueError, cumprod, x, axis=3) self.assertRaises(ValueError, cumprod, x, axis=-4) f = theano.function([x], [cumsum(x), cumprod(x)]) s, p = f(a) assert np.allclose(np.cumsum(a), s) # Test axis=None assert np.allclose(np.cumprod(a), p) # Test axis=None for axis in range(-len(a.shape), len(a.shape)): f = theano.function([x], [cumsum(x, axis=axis), cumprod(x, axis=axis)]) s, p = f(a) assert np.allclose(np.cumsum(a, axis=axis), s) assert np.allclose(np.cumprod(a, axis=axis), p)
def my_keras_cumsum(tensor, axis=0): """ Keras doesn't have a cumsum operation yet, but it seems to be nearly there - see this PR: https://github.com/fchollet/keras/pull/3791. """ # Putting the imports here so that tests and things don't have to run them, unless # they're using this encoder with the particular backend. if K.backend() == "tensorflow": import tensorflow as tf return tf.cumsum(tensor, axis=axis) else: import theano.tensor.extra_ops as T return T.cumsum(tensor, axis=axis)
def test_GpuCumsum3D(self): block_max_size = self.max_threads_dim0 * 2 x = T.ftensor3('x') for shape_axis, axis in zip([0, 1, 2, 0, 2, 1, 0], [0, 1, 2, None, -1, -2, -3]): f = theano.function([x], cumsum(x, axis=axis), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] # Extensive testing for the first 1025 sizes a_shape = [5, 5, 5] a_shape[shape_axis] = 1025 a = np.random.rand(*a_shape).astype("float32") slices = [slice(None), slice(None), slice(None)] for i in xrange(a.shape[shape_axis]): slices[shape_axis] = slice(i) fa = f(a[slices]) npa = np.cumsum(a[slices], axis=axis) utt.assert_allclose(npa, fa) # Use multiple GPU threadblocks (along accumulation axis) a_shape = [2, 2, 2] a_shape[shape_axis] = block_max_size + 2 a = np.random.random(a_shape).astype("float32") utt.assert_allclose(np.cumsum(a, axis=axis), f(a)) # Use multiple GPU gridblocks (not along accumulation axis) a_shape = [5, 5, 5] a_shape[(shape_axis + 1) % 3] = self.max_grid_size1 + 1 a = np.random.random(a_shape).astype("float32") if axis is None: # Avoid floating point error a = np.sign(a - 0.5).astype("float32") utt.assert_allclose(np.cumsum(a, axis=axis), f(a)) a_shape = [5, 5, 5] a_shape[(shape_axis + 2) % 3] = self.max_grid_size1 + 1 a = np.random.random(a_shape).astype("float32") if axis is None: # Avoid floating point error a = np.sign(a - 0.5).astype("float32") utt.assert_allclose(np.cumsum(a, axis=axis), f(a)) # Use recursive cumsum (along accumulation axis) a_shape = [3, 3, 3] a_shape[shape_axis] = block_max_size * ( block_max_size + 1) + 2 a = np.random.random(a_shape).astype("float32") a = np.sign(a - 0.5).astype( "float32") # Avoid floating point error utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
def test_Strides1D(self): x = T.fvector('x') # Stepped strides f = theano.function([x], cumsum(x[::2]), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] a = np.random.randint(10, size=(42,)).astype("float32") assert np.allclose(np.cumsum(a[::2]), f(a)) # Alternative stepped strides f = theano.function([x], cumsum(x), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] a = np.random.randint(10, size=(42,)).astype("float32") assert np.allclose(np.cumsum(a[::2]), f(a[::2])) # Negative strides f = theano.function([x], cumsum(x[::-1]), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] a = np.random.randint(10, size=(42,)).astype("float32") assert np.allclose(np.cumsum(a[::-1]), f(a))
def test_GpuCumsum3D(self): block_max_size = self.max_threads_dim0 * 2 x = T.ftensor3('x') for shape_axis, axis in zip([0, 1, 2, 0, 2, 1, 0], [0, 1, 2, None, -1, -2, -3]): f = theano.function([x], cumsum(x, axis=axis), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] # Extensive testing for the first 1025 sizes a_shape = [5, 5, 5] a_shape[shape_axis] = 1025 a = np.random.rand(*a_shape).astype("float32") slices = [slice(None), slice(None), slice(None)] for i in xrange(a.shape[shape_axis]): slices[shape_axis] = slice(i) fa = f(a[slices]) npa = np.cumsum(a[slices], axis=axis) utt.assert_allclose(npa, fa) # Use multiple GPU threadblocks (along accumulation axis) a_shape = [2, 2, 2] a_shape[shape_axis] = block_max_size + 2 a = np.random.random(a_shape).astype("float32") utt.assert_allclose(np.cumsum(a, axis=axis), f(a)) # Use multiple GPU gridblocks (not along accumulation axis) a_shape = [5, 5, 5] a_shape[(shape_axis + 1) % 3] = self.max_grid_size1 + 1 a = np.random.random(a_shape).astype("float32") if axis is None: # Avoid floating point error a = np.sign(a - 0.5).astype("float32") utt.assert_allclose(np.cumsum(a, axis=axis), f(a)) a_shape = [5, 5, 5] a_shape[(shape_axis + 2) % 3] = self.max_grid_size1 + 1 a = np.random.random(a_shape).astype("float32") if axis is None: # Avoid floating point error a = np.sign(a - 0.5).astype("float32") utt.assert_allclose(np.cumsum(a, axis=axis), f(a)) # Use recursive cumsum (along accumulation axis) a_shape = [3, 3, 3] a_shape[shape_axis] = block_max_size * (block_max_size + 1) + 2 a = np.random.random(a_shape).astype("float32") a = np.sign(a - 0.5).astype("float32") # Avoid floating point error utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
def test_infer_shape(self): # GpuCumSum is only defined for float32 for now, so we skip it # in the unsupported cases gpucumsum_supported_dtypes = ('float32',) if theano.config.floatX not in gpucumsum_supported_dtypes: raise SkipTest('GpuCumSum not implemented for dtype %s' % theano.config.floatX) x = T.tensor3('x') a = np.random.random((3, 5, 2)).astype(theano.config.floatX) for axis in range(-len(a.shape), len(a.shape)): self._compile_and_check([x], [cumsum(x, axis=axis)], [a], self.op_class)
def test_infer_shape(self): x = T.tensor3('x') a = np.random.random((3, 5, 2)).astype(config.floatX) # Test axis=None self._compile_and_check([x], [self.op(x)], [a], self.op_class) for axis in range(-len(a.shape), len(a.shape)): self._compile_and_check([x], [cumsum(x, axis=axis)], [a], self.op_class)
def test_GpuCumsum2D(self): block_max_size = self.max_threads_dim0 * 2 x = T.fmatrix('x') for shape_axis, axis in zip([0, 1, 0], [0, 1, None]): f = theano.function([x], cumsum(x, axis=axis), mode=self.mode) assert [ n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum) ] # Extensive testing for the first 1025 sizes a_shape = [5, 5] a_shape[shape_axis] = 1025 a = np.random.random(a_shape).astype("float32") slices = [slice(None), slice(None)] for i in xrange(a.shape[shape_axis]): slices[shape_axis] = slice(i) fa = f(a[slices]) npa = np.cumsum(a[slices], axis=axis) assert np.allclose(npa, fa) # Use multiple GPU threadblocks a_shape = [5, 5] a_shape[shape_axis] = block_max_size + 2 a = np.random.random(a_shape).astype("float32") assert np.allclose(np.cumsum(a, axis=axis), f(a)) # Use multiple GPU gridblocks a_shape = [5, 5] a_shape[1 - shape_axis] = self.max_grid_size1 + 1 a = np.random.random(a_shape).astype("float32") assert np.allclose(np.cumsum(a, axis=axis), f(a)) # Use recursive cumsum a_shape = [3, 3] a_shape[shape_axis] = block_max_size * (block_max_size + 1) + 2 a = np.random.random(a_shape).astype("float32") a = np.sign(a - 0.5).astype( "float32") # Avoid floating point error assert np.allclose(np.cumsum(a, axis=axis), f(a))
def test_GpuCumsum1D(self): block_max_size = self.max_threads_dim0 * 2 x = T.fvector('x') f = theano.function([x], cumsum(x), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] # Extensive testing for the first 1025 sizes a = np.random.random(1025).astype("float32") for i in xrange(a.shape[0]): utt.assert_allclose(np.cumsum(a[:i]), f(a[:i])) # Use multiple GPU threadblocks a = np.random.random((block_max_size + 2, )).astype("float32") utt.assert_allclose(np.cumsum(a), f(a)) # Use recursive cumsum a = np.ones((block_max_size * (block_max_size + 1) + 2,), dtype="float32") utt.assert_allclose(np.cumsum(a), f(a))
def test_GpuCumsum1D(self): block_max_size = self.max_threads_dim0 * 2 x = T.fvector('x') f = theano.function([x], cumsum(x), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] # Extensive testing for the first 1025 sizes a = np.random.random(1025).astype("float32") for i in xrange(a.shape[0]): utt.assert_allclose(np.cumsum(a[:i]), f(a[:i])) # Use multiple GPU threadblocks a = np.random.random((block_max_size + 2, )).astype("float32") utt.assert_allclose(np.cumsum(a), f(a)) # Use recursive cumsum a = np.ones((block_max_size * (block_max_size + 1) + 2,), dtype="float32") utt.assert_allclose(np.cumsum(a), f(a))
def cost(self, observed, at_risk): """Calculates the cox negative log likelihood. Args: observed: 1D array. Event status; 0 means censored. at_risk: 1D array. Element i of this array indicates the index of the first patient in the at risk group of patient i, when patients are sorted by increasing time to event. Returns: Objective function to be maximized. """ prediction = self.output # Subtracts maximum to facilitate computation. factorizedPred = prediction - prediction.max() exp = T.exp(factorizedPred)[::-1] # Calculates the reversed partial cumulative sum. partial_sum = Te.cumsum(exp)[::-1] + 1 # Adds the subtracted maximum back. log_at_risk = T.log(partial_sum[at_risk]) + prediction.max() diff = prediction - log_at_risk cost = T.sum(T.dot(observed, diff)) return cost
def test_GpuCumsum2D(self): block_max_size = self.max_threads_dim0 * 2 x = T.fmatrix('x') for shape_axis, axis in zip([0, 1, 0], [0, 1, None]): f = theano.function([x], cumsum(x, axis=axis), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] # Extensive testing for the first 1025 sizes a_shape = [5, 5] a_shape[shape_axis] = 1025 a = np.random.random(a_shape).astype("float32") slices = [slice(None), slice(None)] for i in xrange(a.shape[shape_axis]): slices[shape_axis] = slice(i) fa = f(a[slices]) npa = np.cumsum(a[slices], axis=axis) utt.assert_allclose(npa, fa) # Use multiple GPU threadblocks a_shape = [5, 5] a_shape[shape_axis] = block_max_size+2 a = np.random.random(a_shape).astype("float32") utt.assert_allclose(np.cumsum(a, axis=axis), f(a)) # Use multiple GPU gridblocks a_shape = [5, 5] a_shape[1-shape_axis] = self.max_grid_size1+1 a = np.random.random(a_shape).astype("float32") utt.assert_allclose(np.cumsum(a, axis=axis), f(a), rtol=5e-5) # Use recursive cumsum a_shape = [3, 3] a_shape[shape_axis] = block_max_size*(block_max_size+1)+2 a = np.random.random(a_shape).astype("float32") a = np.sign(a-0.5).astype("float32") # Avoid floating point error utt.assert_allclose(np.cumsum(a, axis=axis), f(a))
def _collect_samples(self, y): """ This function collect N samples of size T using the current policy. :param y: :return: locations (n_batch, N, T, 2), probabilities (n_batch, N, T, n_classes), rewards (n_batch, N, T, ) and returns (n_batch, N, T, ) """ means = [] locs = [] probs = [] returns = [] preds = [] # Reshape target labels to match the classification outputs along each path of length T y_rep = T.stack([ T.fill(T.zeros((self.policy.n_steps)), y[b]) for b in xrange(self.policy.n_batch) ], axis=0) for _ in xrange(self.policy.N): loc_means_t, locs_t, _, x_ts, p_ts = self.policy.step_forward() locs.append(locs_t) means.append(loc_means_t) probs.append(p_ts) pred = np.argmax(p_ts, axis=2) preds.append(pred) rewards = self._acc_score(pred, y_rep) returns.append(cumsum(rewards, axis=1)) locs = T.stack(locs).dimshuffle(1, 0, *range(2, T.stack(locs).ndim)) means = T.stack(means).dimshuffle(1, 0, *range(2, T.stack(means).ndim)) preds = T.stack(preds).dimshuffle(1, 0, *range(2, T.stack(preds).ndim)) returns = T.stack(returns).dimshuffle(1, 0, *range(2, T.stack(returns).ndim)) return locs, means, preds, returns
def test_Strides2D(self): x = T.fmatrix('x') for shape_axis, axis in zip([0, 1, 0], [0, 1, None]): a = np.random.random((42, 30)).astype("float32") # Stepped strides along axis=0 f = theano.function([x], cumsum(x[::2], axis=axis), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] assert np.allclose(np.cumsum(a[::2], axis=axis), f(a)) # Stepped strides along axis=1 f = theano.function([x], cumsum(x[:, ::2], axis=axis), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] assert np.allclose(np.cumsum(a[:, ::2], axis=axis), f(a)) # Alternative stepped strides along axis=0 f = theano.function([x], cumsum(x), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] assert np.allclose(np.cumsum(a[::2]), f(a[::2])) # Alternative stepped strides along axis=1 f = theano.function([x], cumsum(x), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] assert np.allclose(np.cumsum(a[:, ::2]), f(a[:, ::2])) # Negative strides along axis=0 f = theano.function([x], cumsum(x[::-1], axis=axis), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] assert np.allclose(np.cumsum(a[::-1], axis=axis), f(a)) # Negative strides along axis=1 f = theano.function([x], cumsum(x[:, ::-1], axis=axis), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, GpuCumsum)] assert np.allclose(np.cumsum(a[:, ::-1], axis=axis), f(a))
def test_GpuCumsum4D(self): # Should not use the GPU version. x = T.ftensor4('x') f = theano.function([x], cumsum(x, axis=1), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, CumsumOp)]
) layer2 = ll.LSTMLayer( input = layer_embed.output, n_in = layer_embed.n_out, n_out = 150, backwards=True ) #gets only the output vectors whose indices are equal to the end of token end_indices = T.nonzero(T.eq(T.argmax(x, axis=1),input_dim-1)) #gets only the output vectors whose indices are equal to the start of token start_indices = T.nonzero(T.eq(T.argmax(x,axis=1),input_dim-2)) hl = T.concatenate((layer1.output, layer2.output[::-1]),axis=1) hc = extra.cumsum(hl, axis=0) hsub = hc[end_indices] - hc[start_indices] diff_indices = T.as_tensor_variable(end_indices) - T.as_tensor_variable(start_indices) diff_shuf = diff_indices.flatten().dimshuffle(0, 'x') h = hsub / diff_shuf h_size = (layer1.n_out + layer2.n_out) #relationship between near words layer_c = nl.NNLayer( input = h, n_in = h_size, n_out = 150, activation = T.tanh)
def kl(Y, Y_hat, cost_mask=None, batch_vec=True, cost_matrix=False, sum_tru_time=False, normalize_by_outsize=True): """ Warning: This function expects a sigmoid nonlinearity in the output layer. Returns a batch (vector) of mean across units of KL divergence for each example, KL(P || Q) where P is defined by Y and Q is defined by Y_hat: p log p - p log q + (1-p) log (1-p) - (1-p) log (1-q) For binary p, some terms drop out: - p log q - (1-p) log (1-q) - p log sigmoid(z) - (1-p) log sigmoid(-z) p softplus(-z) + (1-p) softplus(z) Parameters ---------- Y : Variable targets for the sigmoid outputs. Currently Y must be purely binary. If it's not, you'll still get the right gradient, but the value in the monitoring channel will be wrong. Y_hat : Variable predictions made by the sigmoid layer. Y_hat must be generated by fprop, i.e., it must be a symbolic sigmoid. ------- ave : Variable average kl divergence between Y and Y_hat. """ assert hasattr(Y_hat, 'owner') owner = Y_hat.owner assert owner is not None op = owner.op if not hasattr(op, 'scalar_op'): raise ValueError("Expected Y_hat to be generated by an Elemwise " "op, got "+str(op)+" of type "+str(type(op))) assert isinstance(op.scalar_op, TT.nnet.sigm.ScalarSigmoid) z, = owner.inputs z = z.reshape(Y.shape) term_1 = Y * TT.nnet.softplus(-z) term_2 = (1 - Y) * TT.nnet.softplus(z) total = term_1 + term_2 if cost_mask is not None: if cost_mask.ndim != total.ndim: cost_mask = cost_mask.dimshuffle(0, 1, 'x') total = cost_mask * total if not sum_tru_time: if normalize_by_outsize: if cost_matrix: ave = total.sum(-1) / TT.cast((total.shape[2] - 2), "float32") else: if batch_vec: ave = total.sum(0).sum(1) / TT.cast((total.shape[2] - 2), "float32") else: ave = total.sum() / (total.shape[1] * (total.shape[2] - 2)) else: if cost_matrix: ave = total.sum(-1) else: if batch_vec: ave = total.sum(0).sum(1) else: ave = total.sum() / TT.cast(total.shape[1], "float32") else: assert not cost_matrix if normalize_by_outsize: if batch_vec: ave = cumsum(total.sum(-1) / TT.cast((total.shape[2] - 2), "float32"), axis=0)[::-1] else: ave = cumsum(total.sum((1, 2)) / (total.shape[1] * (total.shape[2] - 2)), axis=0)[::-1] else: if batch_vec: ave = cumsum(total.sum(-1), axis=0)[::-1] else: ave = cumsum(total.sum((1, 2)) / TT.cast(total.shape[1], "float32"), axis=0)[::-1] return ave
def test_GpuCumsum4D(self): # Should not use the GPU version. x = T.ftensor4('x') f = theano.function([x], cumsum(x, axis=1), mode=self.mode) assert [n for n in f.maker.fgraph.toposort() if isinstance(n.op, CumsumOp)]
def kl(Y, Y_hat, cost_mask=None, batch_vec=True, cost_matrix=False, sum_tru_time=False, normalize_by_outsize=True): """ Warning: This function expects a sigmoid nonlinearity in the output layer. Returns a batch (vector) of mean across units of KL divergence for each example, KL(P || Q) where P is defined by Y and Q is defined by Y_hat: p log p - p log q + (1-p) log (1-p) - (1-p) log (1-q) For binary p, some terms drop out: - p log q - (1-p) log (1-q) - p log sigmoid(z) - (1-p) log sigmoid(-z) p softplus(-z) + (1-p) softplus(z) Parameters ---------- Y : Variable targets for the sigmoid outputs. Currently Y must be purely binary. If it's not, you'll still get the right gradient, but the value in the monitoring channel will be wrong. Y_hat : Variable predictions made by the sigmoid layer. Y_hat must be generated by fprop, i.e., it must be a symbolic sigmoid. ------- ave : Variable average kl divergence between Y and Y_hat. """ assert hasattr(Y_hat, 'owner') owner = Y_hat.owner assert owner is not None op = owner.op if not hasattr(op, 'scalar_op'): raise ValueError("Expected Y_hat to be generated by an Elemwise " "op, got " + str(op) + " of type " + str(type(op))) assert isinstance(op.scalar_op, TT.nnet.sigm.ScalarSigmoid) z, = owner.inputs z = z.reshape(Y.shape) term_1 = Y * TT.nnet.softplus(-z) term_2 = (1 - Y) * TT.nnet.softplus(z) total = term_1 + term_2 if cost_mask is not None: if cost_mask.ndim != total.ndim: cost_mask = cost_mask.dimshuffle(0, 1, 'x') total = cost_mask * total if not sum_tru_time: if normalize_by_outsize: if cost_matrix: ave = total.sum(-1) / TT.cast((total.shape[2] - 2), "float32") else: if batch_vec: ave = total.sum(0).sum(1) / TT.cast( (total.shape[2] - 2), "float32") else: ave = total.sum() / (total.shape[1] * (total.shape[2] - 2)) else: if cost_matrix: ave = total.sum(-1) else: if batch_vec: ave = total.sum(0).sum(1) else: ave = total.sum() / TT.cast(total.shape[1], "float32") else: assert not cost_matrix if normalize_by_outsize: if batch_vec: ave = cumsum(total.sum(-1) / TT.cast( (total.shape[2] - 2), "float32"), axis=0)[::-1] else: ave = cumsum(total.sum( (1, 2)) / (total.shape[1] * (total.shape[2] - 2)), axis=0)[::-1] else: if batch_vec: ave = cumsum(total.sum(-1), axis=0)[::-1] else: ave = cumsum(total.sum( (1, 2)) / TT.cast(total.shape[1], "float32"), axis=0)[::-1] return ave