def test_argmax_pushdown():
x = matrix()
for sm in [softmax_graph, softmax_legacy]:
# test that the max_and_argmax is pushed down if the max is not used
out = max_and_argmax(sm(exp(tanh(sigmoid(x)))), axis=-1)[1]
fgraph = FunctionGraph([x], [out])
optdb.query(OPT_FAST_RUN).optimize(fgraph)
# print 'AFTER'
# for node in fgraph.toposort():
# print node.op
assert len(fgraph.toposort()) == 1
assert isinstance(fgraph.toposort()[0].op, Argmax)
assert check_stack_trace(fgraph, ops_to_check=Argmax)
x = matrix()
# test that the max_and_argmax is not pushed down if the max is used
out = max_and_argmax(sm(exp(tanh(sigmoid(x)))), axis=-1)[0]
fgraph = FunctionGraph([x], [out])
assert hasattr(fgraph.outputs[0].tag, "trace")
optdb.query(OPT_FAST_RUN).optimize(fgraph)
# print 'AFTER'
# for node in fgraph.toposort():
# print node.op
assert len(fgraph.toposort()) == 3
assert isinstance(fgraph.toposort()[0].op, Elemwise)
assert isinstance(fgraph.toposort()[1].op, Softmax)
assert isinstance(fgraph.toposort()[2].op, CAReduce)
assert isinstance(fgraph.toposort()[2].op.scalar_op,
aesara.scalar.ScalarMaximum)
def test_composite_c_code(self):
"""Make sure this `Op`'s `c_code` works within a `Composite`."""
x = matrix("x")
mode = get_mode("FAST_RUN").including("local_ultra_fast_sigmoid")
f = aesara.function([x], sigmoid(x) + sigmoid(x + 1), mode=mode)
topo = f.maker.fgraph.toposort()
assert isinstance(topo[0].op, Elemwise)
assert isinstance(topo[0].op.scalar_op, Composite)
assert ultra_fast_scalar_sigmoid in set(
node.op for node in topo[0].op.scalar_op.fgraph.toposort())
assert len(topo) == 1
def step(inp, s_prev):
i_t = sigmoid(
dot(inp, self.W_i) + dot(s_prev, self.R_i) + self.b_wi +
self.b_ru)
r_t = sigmoid(
dot(inp, self.W_r) + dot(s_prev, self.R_r) + self.b_wr +
self.b_rr)
h_hat_t = tanh(
dot(inp, self.W_h) +
(r_t * (dot(s_prev, self.R_h) + self.b_rh)) + self.b_wh)
s_curr = ((1.0 - i_t) * h_hat_t) + (i_t * s_prev)
return s_curr
def test_binary_crossentropy_reshape():
# Reported as https://github.com/Theano/Theano/issues/4086
a = tensor4("a")
for c in (
binary_crossentropy(sigmoid(a.reshape((-1, 1))), 1).sum(),
binary_crossentropy(sigmoid(a).reshape((-1, 1)), 1).sum(),
):
ga = aesara.grad(c, a)
# This only works when "specialize" options are included
mode = aesara.compile.get_default_mode().including("fast_run")
fga = aesara.function([a], ga, mode=mode)
utt.assert_allclose(
fga(np.array([[[[30.0]]]], dtype=config.floatX)),
np.zeros((1, 1, 1, 1), dtype=config.floatX),
)
def test_lop_override(self, cls_ofg):
x = vector()
y = 1.0 / (1.0 + exp(-x))
def lop_ov(inps, outs, grads):
(y_, ) = outs
(dedy_, ) = grads
return [2.0 * y_ * (1.0 - y_) * dedy_]
y_, dedy = vector(), vector()
op_lop_ov = cls_ofg([x, y_, dedy], [2.0 * y_ * (1.0 - y_) * dedy])
xx = vector()
yy1 = aet_sum(sigmoid(xx))
gyy1 = 2.0 * grad(yy1, xx)
for ov in [lop_ov, op_lop_ov]:
op = cls_ofg([x], [y], lop_overrides=ov)
yy2 = aet_sum(op(xx))
gyy2 = grad(yy2, xx)
fn = function([xx], [gyy1, gyy2])
xval = np.random.rand(32).astype(config.floatX)
y1val, y2val = fn(xval)
assert np.allclose(y1val, y2val)
def step(x_t, h_tm1, c_tm1):
i_t = sigmoid(
dot(x_t, self.W_i) + dot(h_tm1, self.R_i) + self.b_wi +
self.b_ri)
f_t = sigmoid(
dot(x_t, self.W_f) + dot(h_tm1, self.R_f) + self.b_wf +
self.b_rf)
o_t = sigmoid(
dot(x_t, self.W_o) + dot(h_tm1, self.R_o) + self.b_ro +
self.b_wo)
c_hat_t = tanh(
dot(x_t, self.W_c) + dot(h_tm1, self.R_c) + self.b_wc +
self.b_rc)
c_t = f_t * c_tm1 + i_t * c_hat_t
h_t = o_t * tanh(c_t)
return h_t, c_t
def rnn_step1(
# sequences
x,
ri,
zi,
# outputs_info
h,
):
pre_r = ri + h.dot(U)
pre_z = zi + h.dot(V)
r = sigmoid(pre_r)
z = sigmoid(pre_z)
after_r = r * h
pre_h = x + after_r.dot(W)
new_h = tanh(pre_h)
res_h = z * new_h + (1 - z) * h
return res_h
def test_matches_binary_crossentropy(self):
# Test sigmoid_binary_crossentropy(p, t) ==
# binary_crossentropy(sigmoid(p), t).
pred, target = inputs = vectors("pt")
reference_val = binary_crossentropy(sigmoid(pred), target)
f_reference = aesara.function(inputs, reference_val)
test_val = sigmoid_binary_crossentropy(pred, target)
f_test = aesara.function(inputs, test_val)
test_inputs = self._get_test_inputs()
utt.assert_allclose(f_reference(*test_inputs), f_test(*test_inputs))
def test_nnet():
x = vector("x")
x.tag.test_value = np.r_[1.0, 2.0].astype(config.floatX)
out = sigmoid(x)
fgraph = FunctionGraph([x], [out])
compare_jax_and_py(fgraph, [get_test_value(i) for i in fgraph.inputs])
out = aet_nnet.ultra_fast_sigmoid(x)
fgraph = FunctionGraph([x], [out])
compare_jax_and_py(fgraph, [get_test_value(i) for i in fgraph.inputs])
out = softplus(x)
fgraph = FunctionGraph([x], [out])
compare_jax_and_py(fgraph, [get_test_value(i) for i in fgraph.inputs])
def test_matches_binary_crossentropy(self):
# Test sigmoid_binary_crossentropy(p, t) ==
# binary_crossentropy(sigmoid(p), t).
pred, target = inputs = vectors("pt")
reference_val = binary_crossentropy(sigmoid(pred), target)
f_reference = aesara.function(inputs, reference_val)
test_val = sigmoid_binary_crossentropy(pred, target)
f_test = aesara.function(inputs, test_val)
rng = np.random.default_rng(utt.fetch_seed())
pred, target = rng.standard_normal((2, 50)).astype(config.floatX)
test_inputs = [pred, 1 / (1 + np.exp(-target))]
utt.assert_allclose(f_reference(*test_inputs), f_test(*test_inputs))
def test_local_ultra_fast_sigmoid(self):
x = matrix("x")
s = sigmoid(x)
mode = self.get_mode("local_ultra_fast_sigmoid")
f = aesara.function([x], s, mode=mode)
assert check_stack_trace(f, ops_to_check=sigmoid)
topo = f.maker.fgraph.toposort()
assert len(topo) == 1
assert topo[0].op == sigmoid
mode = self.get_mode().including("local_ultra_fast_sigmoid")
f = aesara.function([x], s, mode=mode)
assert check_stack_trace(f, ops_to_check=ultra_fast_sigmoid)
topo = f.maker.fgraph.toposort()
assert topo[0].op == ultra_fast_sigmoid
assert len(topo) == 1
f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]])
def __init__(
self,
input=None,
target=None,
n_input=1,
n_hidden=1,
n_output=1,
lr=1e-3,
**kw,
):
super().__init__(**kw)
if input is None:
input = dvector("input")
if target is None:
target = dvector("target")
self.input = input
self.target = target
self.lr = shared(lr, "learning_rate")
self.w1 = shared(np.zeros((n_hidden, n_input)), "w1")
self.w2 = shared(np.zeros((n_output, n_hidden)), "w2")
# print self.lr.type
self.hidden = sigmoid(dot(self.w1, self.input))
self.output = dot(self.w2, self.hidden)
self.cost = aet_sum((self.output - self.target)**2)
self.sgd_updates = {
self.w1: self.w1 - self.lr * grad(self.cost, self.w1),
self.w2: self.w2 - self.lr * grad(self.cost, self.w2),
}
self.sgd_step = pfunc(
params=[self.input, self.target],
outputs=[self.output, self.cost],
updates=self.sgd_updates,
)
self.compute_output = pfunc([self.input], self.output)
self.output_from_hidden = pfunc([self.hidden], self.output)
def test_local_hard_sigmoid(self):
x = matrix("x")
s = sigmoid(x)
mode = self.get_mode("local_hard_sigmoid")
f = aesara.function([x], s, mode=mode)
assert check_stack_trace(f, ops_to_check=sigmoid)
topo = f.maker.fgraph.toposort()
assert topo[0].op == sigmoid
assert len(topo) == 1
mode = self.get_mode().including("local_hard_sigmoid")
f = aesara.function([x], s, mode=mode)
topo = f.maker.fgraph.toposort()
assert not any([n.op == sigmoid for n in topo])
f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]])
mode2 = mode.excluding("fusion").excluding("inplace")
f2 = aesara.function([x], s, mode=mode2)
assert check_stack_trace(f2, ops_to_check=clip)
def test_local_ultra_fast_sigmoid(self):
x = matrix("x")
s = sigmoid(x)
mode = self.get_mode("local_ultra_fast_sigmoid")
f = aesara.function([x], s, mode=mode)
assert check_stack_trace(f, ops_to_check=sigmoid)
topo = f.maker.fgraph.toposort()
assert len(topo) == 1
assert topo[0].op == sigmoid
mode = self.get_mode().including("local_ultra_fast_sigmoid")
f = aesara.function([x], s, mode=mode)
assert check_stack_trace(f, ops_to_check=ultra_fast_sigmoid)
topo = f.maker.fgraph.toposort()
assert topo[0].op == ultra_fast_sigmoid
assert len(topo) == 1
s = sigmoid_inplace(x)
f = aesara.function([x], s, mode=mode, accept_inplace=True)
assert check_stack_trace(f, ops_to_check=ultra_fast_sigmoid_inplace)
topo = f.maker.fgraph.toposort()
assert topo[0].op == ultra_fast_sigmoid_inplace
assert len(topo) == 1
def make_grad_func(X):
Z = dot(X, W) + b
H = sigmoid(Z)
cost = H.sum()
g = grad(cost, X)
return aesara.function([X, W, b], g, on_unused_input="ignore")
