def test_argmax_pushdown():
x = matrix()
for sm in [softmax_graph, softmax_legacy]:
# test that the max_and_argmax is pushed down if the max is not used
out = max_and_argmax(sm(exp(tanh(sigmoid(x)))), axis=-1)[1]
fgraph = FunctionGraph([x], [out])
optdb.query(OPT_FAST_RUN).optimize(fgraph)
# print 'AFTER'
# for node in fgraph.toposort():
# print node.op
assert len(fgraph.toposort()) == 1
assert isinstance(fgraph.toposort()[0].op, Argmax)
assert check_stack_trace(fgraph, ops_to_check=Argmax)
x = matrix()
# test that the max_and_argmax is not pushed down if the max is used
out = max_and_argmax(sm(exp(tanh(sigmoid(x)))), axis=-1)[0]
fgraph = FunctionGraph([x], [out])
assert hasattr(fgraph.outputs[0].tag, "trace")
optdb.query(OPT_FAST_RUN).optimize(fgraph)
# print 'AFTER'
# for node in fgraph.toposort():
# print node.op
assert len(fgraph.toposort()) == 3
assert isinstance(fgraph.toposort()[0].op, Elemwise)
assert isinstance(fgraph.toposort()[1].op, Softmax)
assert isinstance(fgraph.toposort()[2].op, CAReduce)
assert isinstance(fgraph.toposort()[2].op.scalar_op,
aesara.scalar.ScalarMaximum)
def test_NoOutputFromInplace():
x = matrix()
y = matrix()
a = dot(x, y)
b = tanh(a)
c = tanh(dot(2 * x, y))
# Ensure that the elemwise op that produces the output is inplace when
# using a mode that does not include the optimization
fct_no_opt = function([x, y], [b, c], mode="FAST_RUN")
op = fct_no_opt.maker.fgraph.outputs[0].owner.op
assert op.destroy_map and 0 in op.destroy_map
op = fct_no_opt.maker.fgraph.outputs[1].owner.op
assert op.destroy_map and 0 in op.destroy_map
# Ensure that the elemwise op that produces the output is not inplace when
# using a mode that includes the optimization
opt = AddFeatureOptimizer(NoOutputFromInplace([1]))
mode_opt = Mode(linker="py", optimizer="fast_run").register((opt, 49.9))
fct_opt = function([x, y], [b, c], mode=mode_opt)
op = fct_opt.maker.fgraph.outputs[0].owner.op
assert op.destroy_map and 0 in op.destroy_map
op = fct_opt.maker.fgraph.outputs[1].owner.op
assert not op.destroy_map or 0 not in op.destroy_map
def test_reallocation():
x = scalar("x")
y = scalar("y")
z = tanh(3 * x + y) + cosh(x + 5 * y)
# The functinality is currently implement for non lazy and non c VM only.
for linker in [
VMLinker(allow_gc=False, lazy=False, use_cloop=False),
VMLinker(allow_gc=True, lazy=False, use_cloop=False),
]:
m = get_mode(Mode(linker=linker))
m = m.excluding("fusion", "inplace")
f = function([x, y], z, name="test_reduce_memory", mode=m)
output = f(1, 2)
assert output
storage_map = f.fn.storage_map
def check_storage(storage_map):
for i in storage_map:
if not isinstance(i, TensorConstant):
keys_copy = list(storage_map.keys())[:]
keys_copy.remove(i)
for o in keys_copy:
if storage_map[i][
0] and storage_map[i][0] is storage_map[o][0]:
return [True, storage_map[o][0]]
return [False, None]
assert check_storage(storage_map)[0]
assert len({id(v) for v in storage_map.values()}) < len(storage_map)
def test_copy_share_memory(self):
x = fscalar("x")
# SharedVariable for tests, one of them has update
y = shared(value=1)
z = shared(value=2)
out = tanh((x + y + 2) / (x + z - 0.2) ** 2)
# Test for different linkers
for mode in ["FAST_RUN", "FAST_COMPILE"]:
ori = function([x], [out], mode=mode, updates={z: z + 1})
cpy = ori.copy(share_memory=True)
# Test if memories shared
storage_map_ori = ori.fn.storage_map
storage_map_cpy = cpy.fn.storage_map
fgraph_cpy = cpy.maker.fgraph
# Assert intermediate and Constants storages are shared.
# and output stoarges are not shared
i_o_variables = fgraph_cpy.inputs + fgraph_cpy.outputs
ori_storages = storage_map_ori.values()
l = [
val
for key, val in storage_map_cpy.items()
if key not in i_o_variables or isinstance(key, Constant)
]
for storage in l:
assert any([storage is s for s in ori_storages])
# Assert storages of SharedVariable without updates are shared
for (input, _1, _2), here, there in zip(
ori.indices, ori.input_storage, cpy.input_storage
):
assert here.data is there.data
def test_composite_elemwise_float16(self):
w = bvector()
x = vector(dtype="float16")
y = fvector()
cz = tanh(x + aet.cast(y, "float16"))
o = (
cz
- cz ** 2
+ aet.cast(x, "int16")
+ aet.cast(x, "float32")
+ aet.cast(w, "float16")
- aet.constant(np.float16(1.0))
)
aesara.function([w, x, y], o, mode=mode_with_gpu)
v = vector(dtype="uint8")
w = vector(dtype="float16")
x = vector(dtype="float16")
y = vector(dtype="float16")
z = vector(dtype="float16")
o = aet.switch(v, mul(w, x, y), z)
aesara.function([v, w, x, y, z], o, mode=mode_with_gpu)
def step(x_t, h_tm1, c_tm1):
i_t = sigmoid(
dot(x_t, self.W_i) + dot(h_tm1, self.R_i) + self.b_wi +
self.b_ri)
f_t = sigmoid(
dot(x_t, self.W_f) + dot(h_tm1, self.R_f) + self.b_wf +
self.b_rf)
o_t = sigmoid(
dot(x_t, self.W_o) + dot(h_tm1, self.R_o) + self.b_ro +
self.b_wo)
c_hat_t = tanh(
dot(x_t, self.W_c) + dot(h_tm1, self.R_c) + self.b_wc +
self.b_rc)
c_t = f_t * c_tm1 + i_t * c_hat_t
h_t = o_t * tanh(c_t)
return h_t, c_t
def test_subgraph_grad():
# Tests that the grad method with no known_grads
# matches what happens if you use successive subgraph_grads
x = fvector("x")
t = fvector("t")
w1 = aesara.shared(np.random.randn(3, 4))
w2 = aesara.shared(np.random.randn(4, 2))
a1 = tanh(dot(x, w1))
a2 = tanh(dot(a1, w2))
cost2 = sqr(a2 - t).sum()
cost2 += sqr(w2.sum())
cost1 = sqr(w1.sum())
params = [[w2], [w1]]
costs = [cost2, cost1]
grad_ends = [[a1], [x]]
inputs = [t, x]
rng = np.random.RandomState([2012, 11, 15])
values = [rng.randn(2), rng.randn(3)]
values = [np.cast[ipt.dtype](value) for ipt, value in zip(inputs, values)]
wrt = [w2, w1]
cost = cost2 + cost1
true_grads = grad(cost, wrt)
true_grads = aesara.function(inputs, true_grads)
true_grads = true_grads(*values)
next_grad = None
param_grads = []
for i in range(2):
param_grad, next_grad = subgraph_grad(wrt=params[i],
end=grad_ends[i],
start=next_grad,
cost=costs[i])
next_grad = OrderedDict(zip(grad_ends[i], next_grad))
param_grads.extend(param_grad)
pgrads = aesara.function(inputs, param_grads)
pgrads = pgrads(*values)
for true_grad, pgrad in zip(true_grads, pgrads):
assert np.sum(np.abs(true_grad - pgrad)) < 0.00001
def ElemwiseOpTime(N, script=False, loops=1000):
x = vector("x")
np.random.seed(1235)
v = np.random.random(N).astype(config.floatX)
f = aesara.function([x], 2 * x + x * x)
f1 = aesara.function([x], tanh(x))
if not script:
if config.openmp:
print("With openmp:")
print("Fast op ", end=" ")
ceapTime = evalTime(f, v, script=script, loops=loops)
if not script:
print("Slow op ", end=" ")
costlyTime = evalTime(f1, v, script=script, loops=loops)
return (ceapTime, costlyTime)
def step(inp, s_prev):
i_t = sigmoid(
dot(inp, self.W_i) + dot(s_prev, self.R_i) + self.b_wi +
self.b_ru)
r_t = sigmoid(
dot(inp, self.W_r) + dot(s_prev, self.R_r) + self.b_wr +
self.b_rr)
h_hat_t = tanh(
dot(inp, self.W_h) +
(r_t * (dot(s_prev, self.R_h) + self.b_rh)) + self.b_wh)
s_curr = ((1.0 - i_t) * h_hat_t) + (i_t * s_prev)
return s_curr
def rnn_step1(
# sequences
x,
ri,
zi,
# outputs_info
h,
):
pre_r = ri + h.dot(U)
pre_z = zi + h.dot(V)
r = nnet.sigmoid(pre_r)
z = nnet.sigmoid(pre_z)
after_r = r * h
pre_h = x + after_r.dot(W)
new_h = tanh(pre_h)
res_h = z * new_h + (1 - z) * h
return res_h
def test_no_output_from_implace():
x = matrix()
y = matrix()
a = dot(x, y)
b = tanh(a)
# Ensure that the elemwise op that produces the output is inplace when
# using a mode that does not include the optimization
fct_no_opt = aesara.function([x, y], b, mode="FAST_RUN")
op = fct_no_opt.maker.fgraph.outputs[0].owner.op
assert hasattr(op, "destroy_map") and 0 in op.destroy_map
# Ensure that the elemwise op that produces the output is not inplace when
# using a mode that includes the optimization
opt = AddFeatureOptimizer(NoOutputFromInplace())
mode_opt = Mode(linker="cvm", optimizer="fast_run").register((opt, 49.9))
fct_opt = aesara.function([x, y], b, mode=mode_opt)
op = fct_opt.maker.fgraph.outputs[0].owner.op
assert not hasattr(op, "destroy_map") or 0 not in op.destroy_map
def step(x_t, h_tm1):
h = tanh(dot(h_tm1, W_hh) + dot(x_t, W_xh) + b_h)
return h
