def local_exp_over_1_plus_exp(fgraph, node):
"""
exp(x)/(1+exp(x)) -> sigm(x)
c/(1+exp(x)) -> c*sigm(-x)
"""
# this optimization should be done for numerical stability
# so we don't care to check client counts
if node.op == true_div:
# find all the exp() terms in the numerator
num, denom = node.inputs
num_exp_x, num_rest, num_neg = partition_num_or_denom(num, is_exp)
denom_1pexp, denom_rest, denom_neg = partition_num_or_denom(
denom, is_1pexp)
sigmoids = []
for t in denom_1pexp:
if t in num_exp_x:
# case: exp(x) /(1+exp(x))
sigmoids.append(sigmoid(t))
del num_exp_x[num_exp_x.index(t)]
else:
# case: 1/(1+exp(x))
sigmoids.append(sigmoid(-t))
copy_stack_trace(node.outputs[0], sigmoids[-1])
if not sigmoids: # we didn't find any. abort
return
# put the new numerator together
new_num = sigmoids + [exp(t) for t in num_exp_x] + num_rest
if len(new_num) == 1:
new_num = new_num[0]
else:
new_num = mul(*new_num)
if num_neg ^ denom_neg:
new_num = -new_num
copy_stack_trace(num, new_num)
if len(denom_rest) == 0:
return [new_num]
elif len(denom_rest) == 1:
out = new_num / denom_rest[0]
else:
out = new_num / mul(*denom_rest)
copy_stack_trace(node.outputs[0], out)
return [out]
def test_composite_elemwise_float16(self):
w = bvector()
x = vector(dtype="float16")
y = fvector()
cz = tanh(x + aet.cast(y, "float16"))
o = (
cz
- cz ** 2
+ aet.cast(x, "int16")
+ aet.cast(x, "float32")
+ aet.cast(w, "float16")
- aet.constant(np.float16(1.0))
)
aesara.function([w, x, y], o, mode=mode_with_gpu)
v = vector(dtype="uint8")
w = vector(dtype="float16")
x = vector(dtype="float16")
y = vector(dtype="float16")
z = vector(dtype="float16")
o = aet.switch(v, mul(w, x, y), z)
aesara.function([v, w, x, y, z], o, mode=mode_with_gpu)
def compute_mul(tree):
"""
Compute the Variable that is the output of a multiplication tree.
This is the inverse of the operation performed by `parse_mul_tree`, i.e.
compute_mul(parse_mul_tree(tree)) == tree.
Parameters
----------
tree
A multiplication tree (as output by `parse_mul_tree`).
Returns
-------
object
A Variable that computes the multiplication represented by the tree.
"""
neg, inputs = tree
if inputs is None:
raise AssertionError(
"Function `compute_mul` found a missing leaf, did you forget to "
"call `simplify_mul` on the tree first?")
elif isinstance(inputs, list):
# Recurse through inputs.
rval = mul(*list(map(compute_mul, inputs)))
else:
rval = inputs
if neg:
rval = -rval
return rval
def infer_shape(self, fgraph, node, inputs_shapes):
if _variable_is_none(node.inputs[1]):
return [(mul(*inputs_shapes[0]),)]
# axis should not be None, so there should be the same number of
# dimensions in the input and output
assert node.inputs[0].ndim == node.outputs[0].ndim
assert inputs_shapes[1] == ()
return [inputs_shapes[0]]
def infer_shape(self, fgraph, node, inputs_shapes):
if _variable_is_none(node.inputs[1]):
# That means axis = None,
# So the array is flattened before being sorted
return [(mul(*inputs_shapes[0]),)]
# axis should not be None
# So there should be the same number of dimensions
# in the input and output
assert node.inputs[0].ndim == node.outputs[0].ndim
assert inputs_shapes[1] == ()
return [inputs_shapes[0]]
def is_neg(var):
"""
Match a variable with the `-x` pattern.
Parameters
----------
var
The Variable to analyze.
Returns
-------
object
`x` if `var` is of the form `-x`, or None otherwise.
"""
var_node = var.owner
if not var_node:
return None
# First match against `neg`.
if var_node.op == neg:
return var_node.inputs[0]
# Then match against a multiplication by -1.
if var_node.op == mul and len(var_node.inputs) >= 2:
for idx, mul_input in enumerate(var_node.inputs):
try:
constant = get_scalar_constant_value(mul_input)
is_minus_1 = np.allclose(constant, -1)
except NotScalarConstantError:
is_minus_1 = False
if is_minus_1:
# Found a multiplication by -1.
if len(var_node.inputs) == 2:
# Only return the other input.
return var_node.inputs[1 - idx]
else:
# Return the multiplication of all other inputs.
return mul(*(var_node.inputs[0:idx] +
var_node.inputs[idx + 1:]))
# No match.
return None
def infer_shape(self, fgraph, node, ishapes):
from aesara.tensor.math import eq, maximum, mul
# inputs[1] can contain at most one value of '-1', meaning the actual
# shape of the output will be automatically computed by reshape, so
# that the total number of elements stays the same.
# TODO: Maybe put that formula here?
# It's not trivial, because we would have to check if the product of
# all the non-minus-one shapes is a divisor of the product of the
# original shapes.
# The following expression leads to cycles in feature_shape,
# because it tries to replace the Shape_i node by the switch
# statement, which depends on Shape_i.
# return [tuple([switch(eq(node.inputs[1][i], -1),
# Shape_i(i)(node.outputs[0]),
# node.inputs[1][i])
# for i in range(self.ndim)]
# )]
# Here, we only simplify if the shape (node.inputs[1]) is a constant,
# ideally it would suffice to check that it is always non-negative.
# If current variable is a scalar and its dimensionality should
# change to self.ndim, then use size 1 for all new dimensions.
if len(ishapes[0]) == 0:
return [(1, ) * self.ndim]
requ = node.inputs[1]
input_size = mul(*ishapes[0])
if isinstance(requ, TensorConstant):
requ = list(requ.data)
requ_part = [ele for ele in requ if ele != -1]
crit = len(requ) - len(requ_part)
if crit == 1 and len(requ_part) > 0:
# If there are both 0 and -1 in requ_size, it is impossible
# to determine a right output, but we can at least prevent
# a division by 0. We do not want to keep a negative
# size here as it could lead to further weird errors
# after other optimizations.
requ_size = mul(*requ_part)
missing = input_size // (1 if requ_size == 0 else requ_size)
for i, ele in enumerate(requ):
if ele == -1:
requ[i] = missing
elif crit == 1: # we reshape to -1
requ = [input_size] if ishapes[0] else [1]
elif crit > 1:
raise ValueError("shape argument to Reshape.perform"
" must have at most one entry equal to -1")
return [requ]
else:
requ = [requ[i] for i in range(self.ndim)]
# since new_dims can have negative value (-1), the
# multiplication of all values should be negated
# to give a positive value.
# To avoid optimization complexity, we avoid checking
# for the case when there are two or more '-1' values.
if self.ndim:
requ_size = -mul(*requ)
# If there are both 0 and -1 in requ_size, it is impossible
# to determine a right output, but we can at least prevent
# a division by 0. We do not want to keep a negative
# size here as it could lead to further weird errors
# after other optimizations.
rest_size = input_size // maximum(requ_size, 1)
return [
tuple([
aet.switch(eq(requ[i], -1), rest_size, requ[i])
for i in range(self.ndim)
])
]
