def test_argmax_pushdown(): x = matrix() for sm in [softmax_graph, softmax_legacy]: # test that the max_and_argmax is pushed down if the max is not used out = max_and_argmax(sm(exp(tanh(sigmoid(x)))), axis=-1)[1] fgraph = FunctionGraph([x], [out]) optdb.query(OPT_FAST_RUN).optimize(fgraph) # print 'AFTER' # for node in fgraph.toposort(): # print node.op assert len(fgraph.toposort()) == 1 assert isinstance(fgraph.toposort()[0].op, Argmax) assert check_stack_trace(fgraph, ops_to_check=Argmax) x = matrix() # test that the max_and_argmax is not pushed down if the max is used out = max_and_argmax(sm(exp(tanh(sigmoid(x)))), axis=-1)[0] fgraph = FunctionGraph([x], [out]) assert hasattr(fgraph.outputs[0].tag, "trace") optdb.query(OPT_FAST_RUN).optimize(fgraph) # print 'AFTER' # for node in fgraph.toposort(): # print node.op assert len(fgraph.toposort()) == 3 assert isinstance(fgraph.toposort()[0].op, Elemwise) assert isinstance(fgraph.toposort()[1].op, Softmax) assert isinstance(fgraph.toposort()[2].op, CAReduce) assert isinstance(fgraph.toposort()[2].op.scalar_op, aesara.scalar.ScalarMaximum)
def test_composite_c_code(self): """Make sure this `Op`'s `c_code` works within a `Composite`.""" x = matrix("x") mode = get_mode("FAST_RUN").including("local_ultra_fast_sigmoid") f = aesara.function([x], sigmoid(x) + sigmoid(x + 1), mode=mode) topo = f.maker.fgraph.toposort() assert isinstance(topo[0].op, Elemwise) assert isinstance(topo[0].op.scalar_op, Composite) assert ultra_fast_scalar_sigmoid in set( node.op for node in topo[0].op.scalar_op.fgraph.toposort()) assert len(topo) == 1
def step(inp, s_prev): i_t = sigmoid( dot(inp, self.W_i) + dot(s_prev, self.R_i) + self.b_wi + self.b_ru) r_t = sigmoid( dot(inp, self.W_r) + dot(s_prev, self.R_r) + self.b_wr + self.b_rr) h_hat_t = tanh( dot(inp, self.W_h) + (r_t * (dot(s_prev, self.R_h) + self.b_rh)) + self.b_wh) s_curr = ((1.0 - i_t) * h_hat_t) + (i_t * s_prev) return s_curr
def test_binary_crossentropy_reshape(): # Reported as https://github.com/Theano/Theano/issues/4086 a = tensor4("a") for c in ( binary_crossentropy(sigmoid(a.reshape((-1, 1))), 1).sum(), binary_crossentropy(sigmoid(a).reshape((-1, 1)), 1).sum(), ): ga = aesara.grad(c, a) # This only works when "specialize" options are included mode = aesara.compile.get_default_mode().including("fast_run") fga = aesara.function([a], ga, mode=mode) utt.assert_allclose( fga(np.array([[[[30.0]]]], dtype=config.floatX)), np.zeros((1, 1, 1, 1), dtype=config.floatX), )
def test_lop_override(self, cls_ofg): x = vector() y = 1.0 / (1.0 + exp(-x)) def lop_ov(inps, outs, grads): (y_, ) = outs (dedy_, ) = grads return [2.0 * y_ * (1.0 - y_) * dedy_] y_, dedy = vector(), vector() op_lop_ov = cls_ofg([x, y_, dedy], [2.0 * y_ * (1.0 - y_) * dedy]) xx = vector() yy1 = aet_sum(sigmoid(xx)) gyy1 = 2.0 * grad(yy1, xx) for ov in [lop_ov, op_lop_ov]: op = cls_ofg([x], [y], lop_overrides=ov) yy2 = aet_sum(op(xx)) gyy2 = grad(yy2, xx) fn = function([xx], [gyy1, gyy2]) xval = np.random.rand(32).astype(config.floatX) y1val, y2val = fn(xval) assert np.allclose(y1val, y2val)
def step(x_t, h_tm1, c_tm1): i_t = sigmoid( dot(x_t, self.W_i) + dot(h_tm1, self.R_i) + self.b_wi + self.b_ri) f_t = sigmoid( dot(x_t, self.W_f) + dot(h_tm1, self.R_f) + self.b_wf + self.b_rf) o_t = sigmoid( dot(x_t, self.W_o) + dot(h_tm1, self.R_o) + self.b_ro + self.b_wo) c_hat_t = tanh( dot(x_t, self.W_c) + dot(h_tm1, self.R_c) + self.b_wc + self.b_rc) c_t = f_t * c_tm1 + i_t * c_hat_t h_t = o_t * tanh(c_t) return h_t, c_t
def rnn_step1( # sequences x, ri, zi, # outputs_info h, ): pre_r = ri + h.dot(U) pre_z = zi + h.dot(V) r = sigmoid(pre_r) z = sigmoid(pre_z) after_r = r * h pre_h = x + after_r.dot(W) new_h = tanh(pre_h) res_h = z * new_h + (1 - z) * h return res_h
def test_matches_binary_crossentropy(self): # Test sigmoid_binary_crossentropy(p, t) == # binary_crossentropy(sigmoid(p), t). pred, target = inputs = vectors("pt") reference_val = binary_crossentropy(sigmoid(pred), target) f_reference = aesara.function(inputs, reference_val) test_val = sigmoid_binary_crossentropy(pred, target) f_test = aesara.function(inputs, test_val) test_inputs = self._get_test_inputs() utt.assert_allclose(f_reference(*test_inputs), f_test(*test_inputs))
def test_nnet(): x = vector("x") x.tag.test_value = np.r_[1.0, 2.0].astype(config.floatX) out = sigmoid(x) fgraph = FunctionGraph([x], [out]) compare_jax_and_py(fgraph, [get_test_value(i) for i in fgraph.inputs]) out = aet_nnet.ultra_fast_sigmoid(x) fgraph = FunctionGraph([x], [out]) compare_jax_and_py(fgraph, [get_test_value(i) for i in fgraph.inputs]) out = softplus(x) fgraph = FunctionGraph([x], [out]) compare_jax_and_py(fgraph, [get_test_value(i) for i in fgraph.inputs])
def test_matches_binary_crossentropy(self): # Test sigmoid_binary_crossentropy(p, t) == # binary_crossentropy(sigmoid(p), t). pred, target = inputs = vectors("pt") reference_val = binary_crossentropy(sigmoid(pred), target) f_reference = aesara.function(inputs, reference_val) test_val = sigmoid_binary_crossentropy(pred, target) f_test = aesara.function(inputs, test_val) rng = np.random.default_rng(utt.fetch_seed()) pred, target = rng.standard_normal((2, 50)).astype(config.floatX) test_inputs = [pred, 1 / (1 + np.exp(-target))] utt.assert_allclose(f_reference(*test_inputs), f_test(*test_inputs))
def test_local_ultra_fast_sigmoid(self): x = matrix("x") s = sigmoid(x) mode = self.get_mode("local_ultra_fast_sigmoid") f = aesara.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=sigmoid) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == sigmoid mode = self.get_mode().including("local_ultra_fast_sigmoid") f = aesara.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=ultra_fast_sigmoid) topo = f.maker.fgraph.toposort() assert topo[0].op == ultra_fast_sigmoid assert len(topo) == 1 f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]])
def __init__( self, input=None, target=None, n_input=1, n_hidden=1, n_output=1, lr=1e-3, **kw, ): super().__init__(**kw) if input is None: input = dvector("input") if target is None: target = dvector("target") self.input = input self.target = target self.lr = shared(lr, "learning_rate") self.w1 = shared(np.zeros((n_hidden, n_input)), "w1") self.w2 = shared(np.zeros((n_output, n_hidden)), "w2") # print self.lr.type self.hidden = sigmoid(dot(self.w1, self.input)) self.output = dot(self.w2, self.hidden) self.cost = aet_sum((self.output - self.target)**2) self.sgd_updates = { self.w1: self.w1 - self.lr * grad(self.cost, self.w1), self.w2: self.w2 - self.lr * grad(self.cost, self.w2), } self.sgd_step = pfunc( params=[self.input, self.target], outputs=[self.output, self.cost], updates=self.sgd_updates, ) self.compute_output = pfunc([self.input], self.output) self.output_from_hidden = pfunc([self.hidden], self.output)
def test_local_hard_sigmoid(self): x = matrix("x") s = sigmoid(x) mode = self.get_mode("local_hard_sigmoid") f = aesara.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=sigmoid) topo = f.maker.fgraph.toposort() assert topo[0].op == sigmoid assert len(topo) == 1 mode = self.get_mode().including("local_hard_sigmoid") f = aesara.function([x], s, mode=mode) topo = f.maker.fgraph.toposort() assert not any([n.op == sigmoid for n in topo]) f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]]) mode2 = mode.excluding("fusion").excluding("inplace") f2 = aesara.function([x], s, mode=mode2) assert check_stack_trace(f2, ops_to_check=clip)
def test_local_ultra_fast_sigmoid(self): x = matrix("x") s = sigmoid(x) mode = self.get_mode("local_ultra_fast_sigmoid") f = aesara.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=sigmoid) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert topo[0].op == sigmoid mode = self.get_mode().including("local_ultra_fast_sigmoid") f = aesara.function([x], s, mode=mode) assert check_stack_trace(f, ops_to_check=ultra_fast_sigmoid) topo = f.maker.fgraph.toposort() assert topo[0].op == ultra_fast_sigmoid assert len(topo) == 1 s = sigmoid_inplace(x) f = aesara.function([x], s, mode=mode, accept_inplace=True) assert check_stack_trace(f, ops_to_check=ultra_fast_sigmoid_inplace) topo = f.maker.fgraph.toposort() assert topo[0].op == ultra_fast_sigmoid_inplace assert len(topo) == 1
def make_grad_func(X): Z = dot(X, W) + b H = sigmoid(Z) cost = H.sum() g = grad(cost, X) return aesara.function([X, W, b], g, on_unused_input="ignore")