def with_linker_inplace(self, linker):
for xsh, ysh in [((5, 5), (5, 5)),
((5, 5), (1, 5)),
((5, 5), (5, 1)),
((1, 1), (1, 1)),
((2, 3, 4, 5), (2, 3, 4, 5)),
((2, 3, 4, 5), (1, 3, 1, 5)),
((2, 3, 4, 5), (1, 1, 1, 1)),
((), ())]:
x = TensorType('float64', [(entry == 1) for entry in xsh])('x')
y = TensorType('float64', [(entry == 1) for entry in ysh])('y')
e = Elemwise(scalar.Add(scalar.transfer_type(0)), {0: 0})(x, y)
f = copy(linker).accept(FunctionGraph([x, y], [e])).make_function()
xv = numpy.asarray(numpy.random.rand(*xsh))
yv = numpy.asarray(numpy.random.rand(*ysh))
zv = xv + yv
f(xv, yv)
self.assertTrue((xv == zv).all())
#test Elemwise.infer_shape
#the Shape op don't implement c_code!
if isinstance(linker, gof.PerformLinker):
x = TensorType('float64', [(entry == 1) for entry in xsh])('x')
y = TensorType('float64', [(entry == 1) for entry in ysh])('y')
e = Elemwise(scalar.Add(scalar.transfer_type(0)), {0: 0})(x, y)
f = copy(linker).accept(FunctionGraph([x,
y], [e.shape])).make_function()
xv = numpy.asarray(numpy.random.rand(*xsh))
yv = numpy.asarray(numpy.random.rand(*ysh))
zv = xv + yv
f(xv, yv)
assert xv.shape == zv.shape
def make_node(self, *inputs):
res = Elemwise.make_node(self, *inputs)
outputs = [GpuArrayType(broadcastable=o.type.broadcastable,
dtype=o.type.dtype)() for o in res.outputs]
inputs = [as_gpuarray_variable(i) for i in inputs]
node = Apply(self, inputs, outputs)
# Try to generate the kernel to catch SupportCodeErrors
try:
inps = [make_argument(i, 'i%d' % (n,)) for n, i in
enumerate(node.inputs)]
scal_ins = [scalar.Scalar(i.dtype) for i in node.inputs]
outs = [make_argument(o, 'o%d' % (n,)) for n, o in
enumerate(node.outputs) if not n in self.inplace_pattern]
scal_out = [scalar.Scalar(o.dtype) for o in node.outputs]
fake_node = Apply(self.scalar_op, [i() for i in scal_ins],
[o() for o in scal_out])
code = self.scalar_op.c_support_code_apply(fake_node, "test")
if code:
raise SupportCodeError(code)
except MethodNotDefined:
pass
try:
support_code = self.scalar_op.c_support_code()
if (support_code.strip() != "#define THEANO_MACRO_MOD(x,y) (x % y)" and
support_code.strip() != ""):
# The macro is fine, the C++ struct is not.
raise SupportCodeError(support_code)
except MethodNotDefined:
pass
return node
def make_node(self, *inputs):
res = Elemwise.make_node(self, *inputs)
outputs = [GpuArrayType(broadcastable=o.type.broadcastable,
dtype=o.type.dtype)() for o in res.outputs]
inputs = [as_gpuarray_variable(i) for i in inputs]
node = Apply(self, inputs, outputs)
# Try to generate the kernel to catch SupportCodeErrors
try:
inps = [make_argument(i, 'i%d' % (n,)) for n, i in
enumerate(node.inputs)]
scal_ins = [scalar.Scalar(i.dtype) for i in node.inputs]
outs = [make_argument(o, 'o%d' % (n,)) for n, o in
enumerate(node.outputs) if not n in self.inplace_pattern]
scal_out = [scalar.Scalar(o.dtype) for o in node.outputs]
fake_node = Apply(self.scalar_op, [i() for i in scal_ins],
[o() for o in scal_out])
code = self.scalar_op.c_support_code_apply(fake_node, "test")
if code:
raise SupportCodeError(code)
except MethodNotDefined:
pass
try:
support_code = self.scalar_op.c_support_code()
if (support_code.strip() != "#define THEANO_MACRO_MOD(x,y) (x % y)" and
support_code.strip() != ""):
# The macro is fine, the C++ struct is not.
raise SupportCodeError(support_code)
except MethodNotDefined:
pass
return node
def test_same_inputs(self):
if not theano.config.cxx:
raise SkipTest("G++ not available, so we need to skip this test.")
x = TensorType('float64', [0, 0])('x')
e = Elemwise(scalar.add)(x, x)
f = gof.CLinker().accept(FunctionGraph([x], [e])).make_function()
xv = numpy.random.rand(2, 2)
zv = xv + xv
assert (f(xv) == zv).all()
def construct(symbol):
symbolname = symbol.__name__
msg = "no_inplace"
n = "Elemwise{%s,%s}" % (symbolname, msg)
rval = Elemwise(scalar_op, name=n,
nfunc_spec=(nfunc and (nfunc, nin, nout)))
if getattr(symbol, '__doc__', False):
rval.__doc__ = symbol.__doc__ + '\n' + rval.__doc__
# for the meaning of this see the ./epydoc script
# it makes epydoc display rval as if it were a function, not an object
rval.__epydoc_asRoutine = symbol
rval.__module__ = 'tensor'
pprint.assign(rval, printing.FunctionPrinter(symbolname))
return rval
def make_node(self, *inputs):
res = Elemwise.make_node(self, *inputs)
outputs = [GpuArrayType(broadcastable=o.type.broadcastable,
dtype=o.type.dtype)() for o in res.outputs]
inputs = [as_gpuarray_variable(i) for i in inputs]
res = Apply(self, inputs, outputs)
# Try to generate the kernel to catch SupportCodeErrors
k = self.generate_kernel(res, 'test')
return res
def test_fill(self):
if not theano.config.cxx:
raise SkipTest("G++ not available, so we need to skip this test.")
x = TensorType('float64', [0, 0])('x')
y = TensorType('float64', [1, 1])('y')
e = Elemwise(scalar.Second(scalar.transfer_type(0)), {0: 0})(x, y)
f = gof.CLinker().accept(FunctionGraph([x, y], [e])).make_function()
xv = numpy.ones((5, 5))
yv = numpy.random.rand(1, 1)
f(xv, yv)
assert (xv == yv).all()
def test_weird_strides(self):
if not theano.config.cxx:
raise SkipTest("G++ not available, so we need to skip this test.")
x = TensorType('float64', [0, 0, 0, 0, 0])('x')
y = TensorType('float64', [0, 0, 0, 0, 0])('y')
e = Elemwise(scalar.add)(x, y)
f = gof.CLinker().accept(FunctionGraph([x, y], [e])).make_function()
xv = numpy.random.rand(2, 2, 2, 2, 2)
yv = numpy.random.rand(2, 2, 2, 2, 2).transpose(4, 0, 3, 1, 2)
zv = xv + yv
assert (f(xv, yv) == zv).all()
def make_node(self, *inputs):
res = Elemwise.make_node(self, *inputs)
outputs = [
GpuArrayType(broadcastable=o.type.broadcastable,
dtype=o.type.dtype)() for o in res.outputs
]
inputs = [as_gpuarray_variable(i) for i in inputs]
res = Apply(self, inputs, outputs)
# Try to generate the kernel to catch SupportCodeErrors
k = self.generate_kernel(res, 'test')
return res
def test_infer_shape(self):
for s_left, s_right in [((5, 6), (5, 6)), ((5, 6), (5, 1)),
((5, 6), (1, 6)), ((5, 1), (5, 6)),
((1, 6), (5, 6)), ((2, 3, 4, 5), (2, 3, 4, 5)),
((2, 3, 4, 5), (2, 3, 1, 5)),
((2, 3, 4, 5), (1, 3, 4, 5)),
((2, 1, 4, 5), (2, 3, 4, 5)),
((2, 3, 4, 1), (2, 3, 4, 5))]:
dtype = theano.config.floatX
t_left = TensorType(dtype, [(entry == 1) for entry in s_left])()
t_right = TensorType(dtype, [(entry == 1) for entry in s_right])()
t_left_val = numpy.zeros(s_left, dtype=dtype)
t_right_val = numpy.zeros(s_right, dtype=dtype)
self._compile_and_check([t_left, t_right],
[Elemwise(scalar.add)(t_left, t_right)],
[t_left_val, t_right_val], Elemwise)
def batch_normalization(inputs, gamma, beta, mean, std, mode="low_mem"):
"""
This function will build the symbolic graph for applying batch normalization
to a set of activations.
Also works on GPUs, but is not optimized using cuDNN.
.. versionadded:: 0.7.1
Parameters
----------
inputs : symbolic tensor
Mini-batch of activations
gamma: symbolic tensor
BN scale parameter, must be of same dimensionality as
inputs and broadcastable against it
beta: symbolic tensor
BN shift parameter, must be of same dimensionality as
inputs and broadcastable against it
mean: symbolic tensor
inputs means, must be of same dimensionality as
inputs and broadcastable against it
std: symbolic tensor
inputs standard deviation, must be of same dimensionality as
inputs and broadcastable against it
mode: 'low_mem' or 'high_mem'
Specify which batch_normalization implementation that will be
used.
As no intermediate representations are stored for the back-propagation,
'low_mem' implementation lower the memory usage, however,
it is 5-10% slower than 'high_mem' implementation. Note that 5-10% computation
time difference compare the batch_normalization operation only, time difference
between implementation is likely to be less important on the full model fprop/bprop.
"""
if mode == "low_mem":
elm_bn = Elemwise(scalar_op=BNComposite(dtype=inputs.dtype))
rval = elm_bn(inputs, mean, std, gamma, beta)
elif mode == "high_mem":
rval = (inputs - mean) * (gamma / std) + beta
else:
raise ValueError('mode must be either "low_mem", "high_mem"')
return rval
# FIXME: still nan at zero
(x, ) = inputs
(atanhc, ) = outputs
(gz, ) = grads
if x.type in complex_types:
raise NotImplementedError()
return [gz * ((x * (1 - x**2))**-1 - atanhc * x**-1)]
def c_support_code(self):
return """
double _atanhc(double x) {
if (x < -1e-5 || 1e-5 < x) return atanh(x) / x;
return 1 + x * x / 3;
}
"""
def c_code(self, node, name, inp, out, sub):
(x, ) = inp
(z, ) = out
if node.inputs[0].type in float_types:
return ("""%(z)s =
_atanhc(%(x)s);""" % locals())
raise NotImplementedError("only floating point is implemented")
scaler_tanhc = Tanhc(upgrade_to_float, name="tanhc")
tanhc = Elemwise(scaler_tanhc, name="Elemwise{tanhc}", nfunc_spec=None)
scaler_tanhc_grad = TanhcGrad(upgrade_to_float, name="tanhc_grad")
scaler_atanhc = Atanhc(upgrade_to_float, name="atanhc")
atanhc = Elemwise(scaler_atanhc, name="Elemwise{atanhc}", nfunc_spec=None)
gz, = grads
return [gz * (1 + scalar.log(x))]
def c_code(self, node, name, inputs, outputs, sub):
x, = inputs
z, = outputs
if node.inputs[0].type in [scalar.float32, scalar.float64]:
return """%(z)s =
%(x)s == 0.0
? 0.0
: %(x)s * log(%(x)s);""" % locals()
raise NotImplementedError('only floatingpoint is implemented')
scalar_xlogx = XlogX(scalar.upgrade_to_float, name='scalar_xlogx')
xlogx = Elemwise(scalar_xlogx, name='xlogx')
class XlogY0(scalar.BinaryScalarOp):
"""
Compute X * log(Y), with special case 0 log(0) = 0.
"""
@staticmethod
def st_impl(x, y):
if x == 0.0:
return 0.0
return x * numpy.log(y)
def impl(self, x, y):
return XlogY0.st_impl(x, y)
def _set_row_mappings(self, Gamma, dir_priors, model):
"""Create maps from Dirichlet priors parameters to rows and slices in the transition matrix.
These maps are needed when a transition matrix isn't simply comprised
of Dirichlet prior rows, but--instead--slices of Dirichlet priors.
Consider the following:
.. code-block:: python
with pm.Model():
d_0_rv = pm.Dirichlet("p_0", np.r_[1, 1])
d_1_rv = pm.Dirichlet("p_1", np.r_[1, 1])
p_0_rv = tt.as_tensor([0, 0, 1])
p_1_rv = tt.zeros(3)
p_1_rv = tt.set_subtensor(p_0_rv[[0, 2]], d_0_rv)
p_2_rv = tt.zeros(3)
p_2_rv = tt.set_subtensor(p_1_rv[[1, 2]], d_1_rv)
P_tt = tt.stack([p_0_rv, p_1_rv, p_2_rv])
The transition matrix `P_tt` has Dirichlet priors in only two of its
three rows, and--even then--they're only present in parts of two rows.
In this example, we need to know that Dirichlet prior 0, i.e. `d_0_rv`,
is mapped to row 1, and prior 1 is mapped to row 2. Furthermore, we
need to know that prior 0 fills columns 0 and 2 in row 1, and prior 1
fills columns 1 and 2 in row 2.
These mappings allow one to embed Dirichlet priors in larger transition
matrices with--for instance--fixed transition behavior.
""" # noqa: E501
# Remove unimportant `Op`s from the transition matrix graph
Gamma = pre_greedy_local_optimizer(
FunctionGraph([], []),
[
OpRemove(Elemwise(ts.Cast(ts.float32))),
OpRemove(Elemwise(ts.Cast(ts.float64))),
OpRemove(Elemwise(ts.identity)),
],
Gamma,
)
# Canonicalize the transition matrix graph
fg = FunctionGraph(
list(graph_inputs([Gamma] + self.dir_priors_untrans)),
[Gamma] + self.dir_priors_untrans,
clone=True,
)
canonicalize_opt = optdb.query(Query(include=["canonicalize"]))
canonicalize_opt.optimize(fg)
Gamma = fg.outputs[0]
dir_priors_untrans = fg.outputs[1:]
fg.disown()
Gamma_DimShuffle = Gamma.owner
if not (isinstance(Gamma_DimShuffle.op, DimShuffle)):
raise TypeError("The transition matrix should be non-time-varying")
Gamma_Join = Gamma_DimShuffle.inputs[0].owner
if not (isinstance(Gamma_Join.op, tt.basic.Join)):
raise TypeError(
"The transition matrix should be comprised of stacked row vectors"
)
Gamma_rows = Gamma_Join.inputs[1:]
self.n_rows = len(Gamma_rows)
# Loop through the rows in the transition matrix's graph and determine
# how our transformed Dirichlet RVs map to this transition matrix.
self.row_remaps = {}
self.row_slices = {}
for i, dim_row in enumerate(Gamma_rows):
if not dim_row.owner:
continue
# By-pass the `DimShuffle`s applied to the `AdvancedIncSubtensor1`
# `Op`s in which we're actually interested
gamma_row = dim_row.owner.inputs[0]
if gamma_row in dir_priors_untrans:
# This is a row that's simply a `Dirichlet`
j = dir_priors_untrans.index(gamma_row)
self.row_remaps[j] = i
self.row_slices[j] = slice(None)
if gamma_row.owner.inputs[1] not in dir_priors_untrans:
continue
# Parts of a row set by a `*Subtensor*` `Op` using a full
# `Dirichlet` e.g. `P_row[idx] = dir_rv`
j = dir_priors_untrans.index(gamma_row.owner.inputs[1])
untrans_dirich = dir_priors_untrans[j]
if (
gamma_row.owner
and isinstance(gamma_row.owner.op, AdvancedIncSubtensor1)
and gamma_row.owner.inputs[1] == untrans_dirich
):
self.row_remaps[j] = i
rhand_val = gamma_row.owner.inputs[2]
if not isinstance(rhand_val, TensorConstant):
# TODO: We could allow more types of `idx` (e.g. slices)
# Currently, `idx` can't be something like `2:5`
raise TypeError(
"Only array indexing allowed for mixed"
" Dirichlet/non-Dirichlet rows"
)
self.row_slices[j] = rhand_val.data
elif format == "HFLP":
return float16(X)
# float16 function
# we are using the nvidia cuda function (only works on GPU)
class Float16(UnaryScalarOp):
def impl(self, x):
return numpy.float32(numpy.float16(x))
def c_code(self, node, name, (x, ), (z, ), sub):
return "%(z)s = __half2float(__float2half_rn(%(x)s));" % locals()
float16_scalar = Float16(same_out_nocomplex, name='float16')
float16 = Elemwise(float16_scalar)
# this function simulate the precision and the range of a fixed point
# while working with floats
# NOB = Number Of Bits = bit-width
# NOIB = Number Of Integer Bits = position of the radix point = range
def fixed_point(X, NOB, NOIB):
power = T.cast(2.**(NOB - NOIB), theano.config.floatX) # float !
max = T.cast((2.**NOB) - 1, theano.config.floatX)
value = X * power
value = T.round(value) # rounding
value = T.clip(value, -max, max) # saturation arithmetic
value = value / power
return value
from theano.scalar.basic import UnaryScalarOp, same_out_nocomplex
from theano.tensor.elemwise import Elemwise
# Our own rounding function, that does not set the gradient to 0 like Theano's
class Round3(UnaryScalarOp):
def c_code(self, node, name, (x,), (z,), sub):
return "%(z)s = round(%(x)s);" % locals()
def grad(self, inputs, gout):
(gz,) = gout
return gz,
round3_scalar = Round3(same_out_nocomplex, name='round3')
round3 = Elemwise(round3_scalar)
def hard_sigmoid(x):
return T.clip((x+1.)/2.,0,1)
# The neurons' activations binarization function
# It behaves like the sign function during forward propagation
# And like:
# hard_tanh(x) = 2*hard_sigmoid(x)-1
# during back propagation
def binary_tanh_unit(x):
return 2.*round3(hard_sigmoid(x))-1.
def binary_sigmoid_unit(x):
return round3(hard_sigmoid(x))
# casting is done by compiler
(x, ) = inputs
(z, ) = outputs
type = node.inputs[0].type
if type in float_types:
return '%(z)s = (%(x)s > 0) ? %(x)s : (isnan(%(x)s) ? NAN : 0.);' % locals(
)
if type in int_types:
return "%(z)s = (%(x)s >= 0) ? %(x)s : 0;" % locals()
# TODO: A more elegant way to inject operators into packages?
theano.scalar.relu_pg = ReLU_PG(same_out_nocomplex, name='relu_pg')
np.relu_pg = np.frompyfunc(lambda x: x * (x > 0), 1, 1)
relu_pg = Elemwise(theano.scalar.relu_pg)
# ##################### Load data from CIFAR-10 dataset #######################
# this code assumes the cifar dataset from 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
# has been extracted in current working directory
def unpickle(file):
import cPickle
fo = open(file, 'rb')
dict = cPickle.load(fo)
fo.close()
return dict
def load_data():
from theano.tensor.elemwise import Elemwise
# Our own rounding function that does not set the gradient to 0 like Theano's
class __Round(UnaryScalarOp):
def c_code(self, node, name, inputs, outputs, sub):
x, = inputs
z, = outputs
return "%(z)s = round(%(x)s);" % locals()
def grad(self, inputs, gout):
(gz, ) = gout
return gz,
__round_scalar = __Round(same_out_nocomplex, name='__round')
__round = Elemwise(__round_scalar)
def HardSigmoid(x):
return T.clip((x + 1.) / 2., 0, 1)
# The neurons' activations binarization function
# It behaves like the sign function during forward propagation
# And like:
# hard_tanh(x) = 2*hard_sigmoid(x)-1.
# during back propagation
def BinaryTanh(x):
return 2. * __round(HardSigmoid(x)) - 1.
(z, ) = outputs
type = node.inputs[0].type
if type in float_types:
return '%(z)s = (%(x)s > 0) ? 1. : ((%(x)s < 0) ? -1. : (isnan(%(x)s) ? NAN : 0.));' % locals(
)
if type in int_types:
return "%(z)s = (%(x)s >= 0) ? (%(x)s == 0) ? 0 : 1 : -1;" % locals(
)
raise TypeError() # complex has no sgn
# TODO: A more elegant way to inject operators into packages?
theano.scalar.sgn_pg = Sgn_PG(same_out_nocomplex, name='sgn_pg')
np.sgn_pg = np.sign
sgn_pg = Elemwise(theano.scalar.sgn_pg)
# ##################### Load data from CIFAR-10 dataset #######################
# this code assumes the cifar dataset from 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz'
# has been extracted in current working directory
def unpickle(file):
import cPickle
fo = open(file, 'rb')
dict = cPickle.load(fo)
fo.close()
return dict
def load_data():
v_t = b2 * opt_tparams[_p(k, 'v')] + (1. - b2) * g**2
g_t = lr_t * m_t / (tensor.sqrt(v_t) + eps)
p_t = p - g_t
updates[opt_tparams[_p(k, 'm')]] = m_t
updates[opt_tparams[_p(k, 'v')]] = v_t
updates[p] = p_t
updates[opt_tparams['i']] = i_t
return updates, opt_tparams
# Theano Op
class Round(UnaryScalarOp):
def c_code(self, node, name, (x, ), (z, ), sub):
return "%(z)s = round(%(x)s);" % locals()
def grad(self, inputs, gout):
(gz, ) = gout
return gz,
round_scalar = Round(same_out_nocomplex, name='round')
round = Elemwise(round_scalar)
def hard_sigmoid(x, scale=1.):
return tensor.clip((scale * x + 1.) / 2., 0, 1)
def binary_sigmoid(x, scale=1.):
return round(hard_sigmoid(x, scale))
'Binarize',
]
class BinaryOp(UnaryScalarOp):
def c_code(self, node, name, x1, z1, sub):
x = x1[0]
z = z1[0]
return "%(z)s = round(%(x)s);" % locals()
def grad(self, inputs, gout):
(gz,) = gout
return gz,
binaryScalar = BinaryOp(same_out_nocomplex, name='binaryScalar')
binaryOp = Elemwise(binaryScalar)
@layerhelper
class BinaryConv2d(Conv2d):
"""
A binary convolution layer where
both output and weights are binary.
The input may be binary or not.
"""
debugname = 'bindaryconv2d'
LayerTypeName = 'BinaryConv2d'
yaml_tag = u'!BinaryConv2d'
def __init__(self,
filter_size=(3,3),
(gz, ) = grads
return [gz * (1 + scalar.log(x))]
def c_code(self, node, name, inputs, outputs, sub):
(x, ) = inputs
(z, ) = outputs
if node.inputs[0].type in [scalar.float32, scalar.float64]:
return ("""%(z)s =
%(x)s == 0.0
? 0.0
: %(x)s * log(%(x)s);""" % locals())
raise NotImplementedError("only floatingpoint is implemented")
scalar_xlogx = XlogX(scalar.upgrade_to_float, name="scalar_xlogx")
xlogx = Elemwise(scalar_xlogx, name="xlogx")
class XlogY0(scalar.BinaryScalarOp):
"""
Compute X * log(Y), with special case 0 log(0) = 0.
"""
@staticmethod
def st_impl(x, y):
if x == 0.0:
return 0.0
return x * np.log(y)
def impl(self, x, y):
return XlogY0.st_impl(x, y)
def with_linker(self,
linker,
scalar_op=scalar.add,
dtype="floatX",
pre_scalar_op=None,
test_nan=False,
tensor_op=None):
for xsh, tosum in self.cases:
if dtype == "floatX":
dtype = theano.config.floatX
x = TensorType(dtype, [(entry == 1) for entry in xsh])('x')
d = {}
if pre_scalar_op is not None:
d = {"pre_scalar_op": pre_scalar_op}
if tensor_op is None:
e = as_tensor_variable(self.op(scalar_op, axis=tosum, **d)(x))
else:
e = as_tensor_variable(tensor_op(x, axis=tosum, **d))
if tosum is None:
tosum = range(len(xsh))
f = copy(linker).accept(FunctionGraph([x], [e])).make_function()
xv = numpy.asarray(numpy.random.rand(*xsh))
if not "int" in dtype:
xv = numpy.asarray(xv, dtype=dtype)
else:
xv = numpy.asarray(xv < 0.5, dtype=dtype)
if test_nan and xv.size > 0:
if len(xsh) > 0:
xv = xv.flatten()
xv[0] = numpy.nan
xv = xv.reshape(*xsh)
else:
xv = numpy.asarray(numpy.nan, dtype=dtype)
zv = xv
if pre_scalar_op is not None:
zv = Elemwise(scalar_op=pre_scalar_op)(x).eval({x: xv})
numpy_raised = False
if len(tosum) > 1 and any([a < 0 for a in tosum]):
#In that case, we need to use the good order of axis
#in the reduction.
axis2 = []
for a in tosum:
if a < 0:
axis2.append(a + len(xsh))
else:
axis2.append(a)
assert len(axis2) == len(tosum)
tosum = tuple(axis2)
if tensor_op == tensor.all:
for axis in reversed(sorted(tosum)):
zv = numpy.all(zv, axis)
if len(tosum) == 0:
zv = zv != 0
elif tensor_op == tensor.any:
for axis in reversed(sorted(tosum)):
zv = numpy.any(zv, axis)
if len(tosum) == 0:
zv = zv != 0
elif scalar_op == scalar.add:
for axis in reversed(sorted(tosum)):
zv = numpy.add.reduce(zv, axis)
elif scalar_op == scalar.mul:
for axis in reversed(sorted(tosum)):
zv = numpy.multiply.reduce(zv, axis)
elif scalar_op == scalar.maximum:
try:
for axis in reversed(sorted(tosum)):
zv = numpy.maximum.reduce(zv, axis)
except ValueError:
numpy_raised = True
elif scalar_op == scalar.minimum:
try:
for axis in reversed(sorted(tosum)):
zv = numpy.minimum.reduce(zv, axis)
except ValueError:
numpy_raised = True
elif scalar_op == scalar.or_:
for axis in reversed(sorted(tosum)):
zv = numpy.bitwise_or.reduce(zv, axis)
elif scalar_op == scalar.and_:
for axis in reversed(sorted(tosum)):
zv = numpy.bitwise_and.reduce(zv, axis)
elif scalar_op == scalar.xor:
# There is no identity value for the xor function
# So we can't support shape of dimensions 0.
if numpy.prod(zv.shape) == 0:
continue
for axis in reversed(sorted(tosum)):
zv = numpy.bitwise_xor.reduce(zv, axis)
else:
raise Exception(
"Test for CAReduce with scalar_op %s not implemented" %
str(scalar_op))
if scalar_op in [scalar.maximum, scalar.minimum] and numpy_raised:
try:
out = f(xv)
assert out.dtype == dtype
except ValueError:
pass
else:
self.fail()
else:
# numpy.{all,any} return bool type,
# but theano ops return an int8 array instead
if scalar_op in [scalar.and_, scalar.or_]:
zv = numpy.asarray(zv, dtype='int8')
if test_nan:
try:
self.assertTrue(
theano.tensor.TensorType.values_eq(f(xv), zv),
(f(xv), zv))
except NotImplementedError:
# GpuCAReduce don't implement all cases when size is 0
assert xv.size == 0
else:
try:
f_xv = f(xv)
self.assertTrue((f_xv.shape == zv.shape), (f_xv, zv))
self.assertTrue(numpy.allclose(f_xv, zv),
(f_xv, zv, xsh, tosum))
except NotImplementedError:
# GpuCAReduce don't implement all cases when size is 0
assert xv.size == 0
x = TensorType(dtype, [(entry == 1) for entry in xsh])('x')
if tensor_op is None:
e = self.op(scalar_op, axis=tosum)(x)
else:
e = tensor_op(x, axis=tosum)
if tosum is None:
tosum = range(len(xsh))
f = copy(linker).accept(FunctionGraph([x],
[e.shape])).make_function()
if not (scalar_op in [scalar.maximum, scalar.minimum] and
((xsh == () or numpy.prod(xsh) == 0))):
try:
assert all(f(xv) == zv.shape)
except NotImplementedError:
# GpuCAReduce don't implement all cases when size is 0
assert xv.size == 0
from augment import augment_images, adapt_to_binareye
# Our own rounding function, that does not set the gradient to 0 like Theano's
class Round3(UnaryScalarOp):
def c_code(self, node, name, (x, ), (z, ), sub):
return "%(z)s = round(%(x)s);" % locals()
def grad(self, inputs, gout):
(gz, ) = gout
return gz,
round3_scalar = Round3(same_out_nocomplex, name='round3')
round3 = Elemwise(round3_scalar)
def hard_sigmoid(x):
return T.clip((x + 1.) / 2., 0, 1)
# The neurons' activations binarization function
# It behaves like the sign function during forward propagation
# And like:
# hard_tanh(x) = 2*hard_sigmoid(x)-1
# during back propagation
def binary_tanh_unit(x):
return 2. * round3(hard_sigmoid(x)) - 1.
def grad(self, inputs, gout):
(gz, ) = gout
return gz,
class Floor3(UnaryScalarOp):
def c_code(self, node, name, (x, ), (z, ), sub):
return "%(z)s = floor(%(x)s+.5);" % locals() # train -.5 and .5
def grad(self, inputs, gout):
(gz, ) = gout
return gz,
round3_scalar = Round3(same_out_nocomplex, name='round3')
round3 = Elemwise(round3_scalar)
floor3_scalar = Floor3(same_out_nocomplex, name='floor3')
floor3 = Elemwise(floor3_scalar)
def hard_sigmoid(x):
return T.clip((x + 1.) / 2., 0, 1)
# The neurons' activations binarization function
# It behaves like the sign function during forward propagation
# And like:
# hard_tanh(x) = 2*hard_sigmoid(x)-1
# during back propagation
from theano.scalar.basic import UnaryScalarOp, same_out_nocomplex
from theano.tensor.elemwise import Elemwise
from itertools import izip
class round_custom(UnaryScalarOp):
def c_code(self, node, name, (x, ), (z, ), sub):
return "%(z)s = round(%(x)s);" % locals()
def grad(self, inputs, gout):
(gz, ) = gout
return gz,
round_scalar = round_custom(same_out_nocomplex, name='round_var')
round_var = Elemwise(round_scalar)
def hard_sigmoid(x):
return T.clip((x + 1.) / 2., 0, 1)
def discrete_neuron_3states(x): #discrete activation with three states
return T.cast(
round_var(hard_sigmoid(2 * (x - 1)) + hard_sigmoid(2 * (x + 1)) - 1),
theano.config.floatX)
# This class extends the Lasagne DenseLayer to support Probabilistic Discretization of Weights
class DenseLayer(
lasagne.layers.DenseLayer
from theano.scalar.basic import UnaryScalarOp, same_out_nocomplex
from theano.tensor.elemwise import Elemwise
class Round3(UnaryScalarOp):
def c_code(self, node, name, x, z, sub):
return "%(z)s = round(%(x)s);" % locals()
def grad(self, inputs, gout):
(gz, ) = gout
return gz,
round3 = Elemwise(Round3(same_out_nocomplex, name='round3'))
