def with_linker_inplace(self, linker): for xsh, ysh in [((5, 5), (5, 5)), ((5, 5), (1, 5)), ((5, 5), (5, 1)), ((1, 1), (1, 1)), ((2, 3, 4, 5), (2, 3, 4, 5)), ((2, 3, 4, 5), (1, 3, 1, 5)), ((2, 3, 4, 5), (1, 1, 1, 1)), ((), ())]: x = TensorType('float64', [(entry == 1) for entry in xsh])('x') y = TensorType('float64', [(entry == 1) for entry in ysh])('y') e = Elemwise(scalar.Add(scalar.transfer_type(0)), {0: 0})(x, y) f = copy(linker).accept(FunctionGraph([x, y], [e])).make_function() xv = numpy.asarray(numpy.random.rand(*xsh)) yv = numpy.asarray(numpy.random.rand(*ysh)) zv = xv + yv f(xv, yv) self.assertTrue((xv == zv).all()) #test Elemwise.infer_shape #the Shape op don't implement c_code! if isinstance(linker, gof.PerformLinker): x = TensorType('float64', [(entry == 1) for entry in xsh])('x') y = TensorType('float64', [(entry == 1) for entry in ysh])('y') e = Elemwise(scalar.Add(scalar.transfer_type(0)), {0: 0})(x, y) f = copy(linker).accept(FunctionGraph([x, y], [e.shape])).make_function() xv = numpy.asarray(numpy.random.rand(*xsh)) yv = numpy.asarray(numpy.random.rand(*ysh)) zv = xv + yv f(xv, yv) assert xv.shape == zv.shape
def make_node(self, *inputs): res = Elemwise.make_node(self, *inputs) outputs = [GpuArrayType(broadcastable=o.type.broadcastable, dtype=o.type.dtype)() for o in res.outputs] inputs = [as_gpuarray_variable(i) for i in inputs] node = Apply(self, inputs, outputs) # Try to generate the kernel to catch SupportCodeErrors try: inps = [make_argument(i, 'i%d' % (n,)) for n, i in enumerate(node.inputs)] scal_ins = [scalar.Scalar(i.dtype) for i in node.inputs] outs = [make_argument(o, 'o%d' % (n,)) for n, o in enumerate(node.outputs) if not n in self.inplace_pattern] scal_out = [scalar.Scalar(o.dtype) for o in node.outputs] fake_node = Apply(self.scalar_op, [i() for i in scal_ins], [o() for o in scal_out]) code = self.scalar_op.c_support_code_apply(fake_node, "test") if code: raise SupportCodeError(code) except MethodNotDefined: pass try: support_code = self.scalar_op.c_support_code() if (support_code.strip() != "#define THEANO_MACRO_MOD(x,y) (x % y)" and support_code.strip() != ""): # The macro is fine, the C++ struct is not. raise SupportCodeError(support_code) except MethodNotDefined: pass return node
def test_same_inputs(self): if not theano.config.cxx: raise SkipTest("G++ not available, so we need to skip this test.") x = TensorType('float64', [0, 0])('x') e = Elemwise(scalar.add)(x, x) f = gof.CLinker().accept(FunctionGraph([x], [e])).make_function() xv = numpy.random.rand(2, 2) zv = xv + xv assert (f(xv) == zv).all()
def construct(symbol): symbolname = symbol.__name__ msg = "no_inplace" n = "Elemwise{%s,%s}" % (symbolname, msg) rval = Elemwise(scalar_op, name=n, nfunc_spec=(nfunc and (nfunc, nin, nout))) if getattr(symbol, '__doc__', False): rval.__doc__ = symbol.__doc__ + '\n' + rval.__doc__ # for the meaning of this see the ./epydoc script # it makes epydoc display rval as if it were a function, not an object rval.__epydoc_asRoutine = symbol rval.__module__ = 'tensor' pprint.assign(rval, printing.FunctionPrinter(symbolname)) return rval
def make_node(self, *inputs): res = Elemwise.make_node(self, *inputs) outputs = [GpuArrayType(broadcastable=o.type.broadcastable, dtype=o.type.dtype)() for o in res.outputs] inputs = [as_gpuarray_variable(i) for i in inputs] res = Apply(self, inputs, outputs) # Try to generate the kernel to catch SupportCodeErrors k = self.generate_kernel(res, 'test') return res
def test_fill(self): if not theano.config.cxx: raise SkipTest("G++ not available, so we need to skip this test.") x = TensorType('float64', [0, 0])('x') y = TensorType('float64', [1, 1])('y') e = Elemwise(scalar.Second(scalar.transfer_type(0)), {0: 0})(x, y) f = gof.CLinker().accept(FunctionGraph([x, y], [e])).make_function() xv = numpy.ones((5, 5)) yv = numpy.random.rand(1, 1) f(xv, yv) assert (xv == yv).all()
def test_weird_strides(self): if not theano.config.cxx: raise SkipTest("G++ not available, so we need to skip this test.") x = TensorType('float64', [0, 0, 0, 0, 0])('x') y = TensorType('float64', [0, 0, 0, 0, 0])('y') e = Elemwise(scalar.add)(x, y) f = gof.CLinker().accept(FunctionGraph([x, y], [e])).make_function() xv = numpy.random.rand(2, 2, 2, 2, 2) yv = numpy.random.rand(2, 2, 2, 2, 2).transpose(4, 0, 3, 1, 2) zv = xv + yv assert (f(xv, yv) == zv).all()
def make_node(self, *inputs): res = Elemwise.make_node(self, *inputs) outputs = [ GpuArrayType(broadcastable=o.type.broadcastable, dtype=o.type.dtype)() for o in res.outputs ] inputs = [as_gpuarray_variable(i) for i in inputs] res = Apply(self, inputs, outputs) # Try to generate the kernel to catch SupportCodeErrors k = self.generate_kernel(res, 'test') return res
def test_infer_shape(self): for s_left, s_right in [((5, 6), (5, 6)), ((5, 6), (5, 1)), ((5, 6), (1, 6)), ((5, 1), (5, 6)), ((1, 6), (5, 6)), ((2, 3, 4, 5), (2, 3, 4, 5)), ((2, 3, 4, 5), (2, 3, 1, 5)), ((2, 3, 4, 5), (1, 3, 4, 5)), ((2, 1, 4, 5), (2, 3, 4, 5)), ((2, 3, 4, 1), (2, 3, 4, 5))]: dtype = theano.config.floatX t_left = TensorType(dtype, [(entry == 1) for entry in s_left])() t_right = TensorType(dtype, [(entry == 1) for entry in s_right])() t_left_val = numpy.zeros(s_left, dtype=dtype) t_right_val = numpy.zeros(s_right, dtype=dtype) self._compile_and_check([t_left, t_right], [Elemwise(scalar.add)(t_left, t_right)], [t_left_val, t_right_val], Elemwise)
def batch_normalization(inputs, gamma, beta, mean, std, mode="low_mem"): """ This function will build the symbolic graph for applying batch normalization to a set of activations. Also works on GPUs, but is not optimized using cuDNN. .. versionadded:: 0.7.1 Parameters ---------- inputs : symbolic tensor Mini-batch of activations gamma: symbolic tensor BN scale parameter, must be of same dimensionality as inputs and broadcastable against it beta: symbolic tensor BN shift parameter, must be of same dimensionality as inputs and broadcastable against it mean: symbolic tensor inputs means, must be of same dimensionality as inputs and broadcastable against it std: symbolic tensor inputs standard deviation, must be of same dimensionality as inputs and broadcastable against it mode: 'low_mem' or 'high_mem' Specify which batch_normalization implementation that will be used. As no intermediate representations are stored for the back-propagation, 'low_mem' implementation lower the memory usage, however, it is 5-10% slower than 'high_mem' implementation. Note that 5-10% computation time difference compare the batch_normalization operation only, time difference between implementation is likely to be less important on the full model fprop/bprop. """ if mode == "low_mem": elm_bn = Elemwise(scalar_op=BNComposite(dtype=inputs.dtype)) rval = elm_bn(inputs, mean, std, gamma, beta) elif mode == "high_mem": rval = (inputs - mean) * (gamma / std) + beta else: raise ValueError('mode must be either "low_mem", "high_mem"') return rval
# FIXME: still nan at zero (x, ) = inputs (atanhc, ) = outputs (gz, ) = grads if x.type in complex_types: raise NotImplementedError() return [gz * ((x * (1 - x**2))**-1 - atanhc * x**-1)] def c_support_code(self): return """ double _atanhc(double x) { if (x < -1e-5 || 1e-5 < x) return atanh(x) / x; return 1 + x * x / 3; } """ def c_code(self, node, name, inp, out, sub): (x, ) = inp (z, ) = out if node.inputs[0].type in float_types: return ("""%(z)s = _atanhc(%(x)s);""" % locals()) raise NotImplementedError("only floating point is implemented") scaler_tanhc = Tanhc(upgrade_to_float, name="tanhc") tanhc = Elemwise(scaler_tanhc, name="Elemwise{tanhc}", nfunc_spec=None) scaler_tanhc_grad = TanhcGrad(upgrade_to_float, name="tanhc_grad") scaler_atanhc = Atanhc(upgrade_to_float, name="atanhc") atanhc = Elemwise(scaler_atanhc, name="Elemwise{atanhc}", nfunc_spec=None)
gz, = grads return [gz * (1 + scalar.log(x))] def c_code(self, node, name, inputs, outputs, sub): x, = inputs z, = outputs if node.inputs[0].type in [scalar.float32, scalar.float64]: return """%(z)s = %(x)s == 0.0 ? 0.0 : %(x)s * log(%(x)s);""" % locals() raise NotImplementedError('only floatingpoint is implemented') scalar_xlogx = XlogX(scalar.upgrade_to_float, name='scalar_xlogx') xlogx = Elemwise(scalar_xlogx, name='xlogx') class XlogY0(scalar.BinaryScalarOp): """ Compute X * log(Y), with special case 0 log(0) = 0. """ @staticmethod def st_impl(x, y): if x == 0.0: return 0.0 return x * numpy.log(y) def impl(self, x, y): return XlogY0.st_impl(x, y)
def _set_row_mappings(self, Gamma, dir_priors, model): """Create maps from Dirichlet priors parameters to rows and slices in the transition matrix. These maps are needed when a transition matrix isn't simply comprised of Dirichlet prior rows, but--instead--slices of Dirichlet priors. Consider the following: .. code-block:: python with pm.Model(): d_0_rv = pm.Dirichlet("p_0", np.r_[1, 1]) d_1_rv = pm.Dirichlet("p_1", np.r_[1, 1]) p_0_rv = tt.as_tensor([0, 0, 1]) p_1_rv = tt.zeros(3) p_1_rv = tt.set_subtensor(p_0_rv[[0, 2]], d_0_rv) p_2_rv = tt.zeros(3) p_2_rv = tt.set_subtensor(p_1_rv[[1, 2]], d_1_rv) P_tt = tt.stack([p_0_rv, p_1_rv, p_2_rv]) The transition matrix `P_tt` has Dirichlet priors in only two of its three rows, and--even then--they're only present in parts of two rows. In this example, we need to know that Dirichlet prior 0, i.e. `d_0_rv`, is mapped to row 1, and prior 1 is mapped to row 2. Furthermore, we need to know that prior 0 fills columns 0 and 2 in row 1, and prior 1 fills columns 1 and 2 in row 2. These mappings allow one to embed Dirichlet priors in larger transition matrices with--for instance--fixed transition behavior. """ # noqa: E501 # Remove unimportant `Op`s from the transition matrix graph Gamma = pre_greedy_local_optimizer( FunctionGraph([], []), [ OpRemove(Elemwise(ts.Cast(ts.float32))), OpRemove(Elemwise(ts.Cast(ts.float64))), OpRemove(Elemwise(ts.identity)), ], Gamma, ) # Canonicalize the transition matrix graph fg = FunctionGraph( list(graph_inputs([Gamma] + self.dir_priors_untrans)), [Gamma] + self.dir_priors_untrans, clone=True, ) canonicalize_opt = optdb.query(Query(include=["canonicalize"])) canonicalize_opt.optimize(fg) Gamma = fg.outputs[0] dir_priors_untrans = fg.outputs[1:] fg.disown() Gamma_DimShuffle = Gamma.owner if not (isinstance(Gamma_DimShuffle.op, DimShuffle)): raise TypeError("The transition matrix should be non-time-varying") Gamma_Join = Gamma_DimShuffle.inputs[0].owner if not (isinstance(Gamma_Join.op, tt.basic.Join)): raise TypeError( "The transition matrix should be comprised of stacked row vectors" ) Gamma_rows = Gamma_Join.inputs[1:] self.n_rows = len(Gamma_rows) # Loop through the rows in the transition matrix's graph and determine # how our transformed Dirichlet RVs map to this transition matrix. self.row_remaps = {} self.row_slices = {} for i, dim_row in enumerate(Gamma_rows): if not dim_row.owner: continue # By-pass the `DimShuffle`s applied to the `AdvancedIncSubtensor1` # `Op`s in which we're actually interested gamma_row = dim_row.owner.inputs[0] if gamma_row in dir_priors_untrans: # This is a row that's simply a `Dirichlet` j = dir_priors_untrans.index(gamma_row) self.row_remaps[j] = i self.row_slices[j] = slice(None) if gamma_row.owner.inputs[1] not in dir_priors_untrans: continue # Parts of a row set by a `*Subtensor*` `Op` using a full # `Dirichlet` e.g. `P_row[idx] = dir_rv` j = dir_priors_untrans.index(gamma_row.owner.inputs[1]) untrans_dirich = dir_priors_untrans[j] if ( gamma_row.owner and isinstance(gamma_row.owner.op, AdvancedIncSubtensor1) and gamma_row.owner.inputs[1] == untrans_dirich ): self.row_remaps[j] = i rhand_val = gamma_row.owner.inputs[2] if not isinstance(rhand_val, TensorConstant): # TODO: We could allow more types of `idx` (e.g. slices) # Currently, `idx` can't be something like `2:5` raise TypeError( "Only array indexing allowed for mixed" " Dirichlet/non-Dirichlet rows" ) self.row_slices[j] = rhand_val.data
elif format == "HFLP": return float16(X) # float16 function # we are using the nvidia cuda function (only works on GPU) class Float16(UnaryScalarOp): def impl(self, x): return numpy.float32(numpy.float16(x)) def c_code(self, node, name, (x, ), (z, ), sub): return "%(z)s = __half2float(__float2half_rn(%(x)s));" % locals() float16_scalar = Float16(same_out_nocomplex, name='float16') float16 = Elemwise(float16_scalar) # this function simulate the precision and the range of a fixed point # while working with floats # NOB = Number Of Bits = bit-width # NOIB = Number Of Integer Bits = position of the radix point = range def fixed_point(X, NOB, NOIB): power = T.cast(2.**(NOB - NOIB), theano.config.floatX) # float ! max = T.cast((2.**NOB) - 1, theano.config.floatX) value = X * power value = T.round(value) # rounding value = T.clip(value, -max, max) # saturation arithmetic value = value / power return value
from theano.scalar.basic import UnaryScalarOp, same_out_nocomplex from theano.tensor.elemwise import Elemwise # Our own rounding function, that does not set the gradient to 0 like Theano's class Round3(UnaryScalarOp): def c_code(self, node, name, (x,), (z,), sub): return "%(z)s = round(%(x)s);" % locals() def grad(self, inputs, gout): (gz,) = gout return gz, round3_scalar = Round3(same_out_nocomplex, name='round3') round3 = Elemwise(round3_scalar) def hard_sigmoid(x): return T.clip((x+1.)/2.,0,1) # The neurons' activations binarization function # It behaves like the sign function during forward propagation # And like: # hard_tanh(x) = 2*hard_sigmoid(x)-1 # during back propagation def binary_tanh_unit(x): return 2.*round3(hard_sigmoid(x))-1. def binary_sigmoid_unit(x): return round3(hard_sigmoid(x))
# casting is done by compiler (x, ) = inputs (z, ) = outputs type = node.inputs[0].type if type in float_types: return '%(z)s = (%(x)s > 0) ? %(x)s : (isnan(%(x)s) ? NAN : 0.);' % locals( ) if type in int_types: return "%(z)s = (%(x)s >= 0) ? %(x)s : 0;" % locals() # TODO: A more elegant way to inject operators into packages? theano.scalar.relu_pg = ReLU_PG(same_out_nocomplex, name='relu_pg') np.relu_pg = np.frompyfunc(lambda x: x * (x > 0), 1, 1) relu_pg = Elemwise(theano.scalar.relu_pg) # ##################### Load data from CIFAR-10 dataset ####################### # this code assumes the cifar dataset from 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz' # has been extracted in current working directory def unpickle(file): import cPickle fo = open(file, 'rb') dict = cPickle.load(fo) fo.close() return dict def load_data():
from theano.tensor.elemwise import Elemwise # Our own rounding function that does not set the gradient to 0 like Theano's class __Round(UnaryScalarOp): def c_code(self, node, name, inputs, outputs, sub): x, = inputs z, = outputs return "%(z)s = round(%(x)s);" % locals() def grad(self, inputs, gout): (gz, ) = gout return gz, __round_scalar = __Round(same_out_nocomplex, name='__round') __round = Elemwise(__round_scalar) def HardSigmoid(x): return T.clip((x + 1.) / 2., 0, 1) # The neurons' activations binarization function # It behaves like the sign function during forward propagation # And like: # hard_tanh(x) = 2*hard_sigmoid(x)-1. # during back propagation def BinaryTanh(x): return 2. * __round(HardSigmoid(x)) - 1.
(z, ) = outputs type = node.inputs[0].type if type in float_types: return '%(z)s = (%(x)s > 0) ? 1. : ((%(x)s < 0) ? -1. : (isnan(%(x)s) ? NAN : 0.));' % locals( ) if type in int_types: return "%(z)s = (%(x)s >= 0) ? (%(x)s == 0) ? 0 : 1 : -1;" % locals( ) raise TypeError() # complex has no sgn # TODO: A more elegant way to inject operators into packages? theano.scalar.sgn_pg = Sgn_PG(same_out_nocomplex, name='sgn_pg') np.sgn_pg = np.sign sgn_pg = Elemwise(theano.scalar.sgn_pg) # ##################### Load data from CIFAR-10 dataset ####################### # this code assumes the cifar dataset from 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz' # has been extracted in current working directory def unpickle(file): import cPickle fo = open(file, 'rb') dict = cPickle.load(fo) fo.close() return dict def load_data():
v_t = b2 * opt_tparams[_p(k, 'v')] + (1. - b2) * g**2 g_t = lr_t * m_t / (tensor.sqrt(v_t) + eps) p_t = p - g_t updates[opt_tparams[_p(k, 'm')]] = m_t updates[opt_tparams[_p(k, 'v')]] = v_t updates[p] = p_t updates[opt_tparams['i']] = i_t return updates, opt_tparams # Theano Op class Round(UnaryScalarOp): def c_code(self, node, name, (x, ), (z, ), sub): return "%(z)s = round(%(x)s);" % locals() def grad(self, inputs, gout): (gz, ) = gout return gz, round_scalar = Round(same_out_nocomplex, name='round') round = Elemwise(round_scalar) def hard_sigmoid(x, scale=1.): return tensor.clip((scale * x + 1.) / 2., 0, 1) def binary_sigmoid(x, scale=1.): return round(hard_sigmoid(x, scale))
'Binarize', ] class BinaryOp(UnaryScalarOp): def c_code(self, node, name, x1, z1, sub): x = x1[0] z = z1[0] return "%(z)s = round(%(x)s);" % locals() def grad(self, inputs, gout): (gz,) = gout return gz, binaryScalar = BinaryOp(same_out_nocomplex, name='binaryScalar') binaryOp = Elemwise(binaryScalar) @layerhelper class BinaryConv2d(Conv2d): """ A binary convolution layer where both output and weights are binary. The input may be binary or not. """ debugname = 'bindaryconv2d' LayerTypeName = 'BinaryConv2d' yaml_tag = u'!BinaryConv2d' def __init__(self, filter_size=(3,3),
(gz, ) = grads return [gz * (1 + scalar.log(x))] def c_code(self, node, name, inputs, outputs, sub): (x, ) = inputs (z, ) = outputs if node.inputs[0].type in [scalar.float32, scalar.float64]: return ("""%(z)s = %(x)s == 0.0 ? 0.0 : %(x)s * log(%(x)s);""" % locals()) raise NotImplementedError("only floatingpoint is implemented") scalar_xlogx = XlogX(scalar.upgrade_to_float, name="scalar_xlogx") xlogx = Elemwise(scalar_xlogx, name="xlogx") class XlogY0(scalar.BinaryScalarOp): """ Compute X * log(Y), with special case 0 log(0) = 0. """ @staticmethod def st_impl(x, y): if x == 0.0: return 0.0 return x * np.log(y) def impl(self, x, y): return XlogY0.st_impl(x, y)
def with_linker(self, linker, scalar_op=scalar.add, dtype="floatX", pre_scalar_op=None, test_nan=False, tensor_op=None): for xsh, tosum in self.cases: if dtype == "floatX": dtype = theano.config.floatX x = TensorType(dtype, [(entry == 1) for entry in xsh])('x') d = {} if pre_scalar_op is not None: d = {"pre_scalar_op": pre_scalar_op} if tensor_op is None: e = as_tensor_variable(self.op(scalar_op, axis=tosum, **d)(x)) else: e = as_tensor_variable(tensor_op(x, axis=tosum, **d)) if tosum is None: tosum = range(len(xsh)) f = copy(linker).accept(FunctionGraph([x], [e])).make_function() xv = numpy.asarray(numpy.random.rand(*xsh)) if not "int" in dtype: xv = numpy.asarray(xv, dtype=dtype) else: xv = numpy.asarray(xv < 0.5, dtype=dtype) if test_nan and xv.size > 0: if len(xsh) > 0: xv = xv.flatten() xv[0] = numpy.nan xv = xv.reshape(*xsh) else: xv = numpy.asarray(numpy.nan, dtype=dtype) zv = xv if pre_scalar_op is not None: zv = Elemwise(scalar_op=pre_scalar_op)(x).eval({x: xv}) numpy_raised = False if len(tosum) > 1 and any([a < 0 for a in tosum]): #In that case, we need to use the good order of axis #in the reduction. axis2 = [] for a in tosum: if a < 0: axis2.append(a + len(xsh)) else: axis2.append(a) assert len(axis2) == len(tosum) tosum = tuple(axis2) if tensor_op == tensor.all: for axis in reversed(sorted(tosum)): zv = numpy.all(zv, axis) if len(tosum) == 0: zv = zv != 0 elif tensor_op == tensor.any: for axis in reversed(sorted(tosum)): zv = numpy.any(zv, axis) if len(tosum) == 0: zv = zv != 0 elif scalar_op == scalar.add: for axis in reversed(sorted(tosum)): zv = numpy.add.reduce(zv, axis) elif scalar_op == scalar.mul: for axis in reversed(sorted(tosum)): zv = numpy.multiply.reduce(zv, axis) elif scalar_op == scalar.maximum: try: for axis in reversed(sorted(tosum)): zv = numpy.maximum.reduce(zv, axis) except ValueError: numpy_raised = True elif scalar_op == scalar.minimum: try: for axis in reversed(sorted(tosum)): zv = numpy.minimum.reduce(zv, axis) except ValueError: numpy_raised = True elif scalar_op == scalar.or_: for axis in reversed(sorted(tosum)): zv = numpy.bitwise_or.reduce(zv, axis) elif scalar_op == scalar.and_: for axis in reversed(sorted(tosum)): zv = numpy.bitwise_and.reduce(zv, axis) elif scalar_op == scalar.xor: # There is no identity value for the xor function # So we can't support shape of dimensions 0. if numpy.prod(zv.shape) == 0: continue for axis in reversed(sorted(tosum)): zv = numpy.bitwise_xor.reduce(zv, axis) else: raise Exception( "Test for CAReduce with scalar_op %s not implemented" % str(scalar_op)) if scalar_op in [scalar.maximum, scalar.minimum] and numpy_raised: try: out = f(xv) assert out.dtype == dtype except ValueError: pass else: self.fail() else: # numpy.{all,any} return bool type, # but theano ops return an int8 array instead if scalar_op in [scalar.and_, scalar.or_]: zv = numpy.asarray(zv, dtype='int8') if test_nan: try: self.assertTrue( theano.tensor.TensorType.values_eq(f(xv), zv), (f(xv), zv)) except NotImplementedError: # GpuCAReduce don't implement all cases when size is 0 assert xv.size == 0 else: try: f_xv = f(xv) self.assertTrue((f_xv.shape == zv.shape), (f_xv, zv)) self.assertTrue(numpy.allclose(f_xv, zv), (f_xv, zv, xsh, tosum)) except NotImplementedError: # GpuCAReduce don't implement all cases when size is 0 assert xv.size == 0 x = TensorType(dtype, [(entry == 1) for entry in xsh])('x') if tensor_op is None: e = self.op(scalar_op, axis=tosum)(x) else: e = tensor_op(x, axis=tosum) if tosum is None: tosum = range(len(xsh)) f = copy(linker).accept(FunctionGraph([x], [e.shape])).make_function() if not (scalar_op in [scalar.maximum, scalar.minimum] and ((xsh == () or numpy.prod(xsh) == 0))): try: assert all(f(xv) == zv.shape) except NotImplementedError: # GpuCAReduce don't implement all cases when size is 0 assert xv.size == 0
from augment import augment_images, adapt_to_binareye # Our own rounding function, that does not set the gradient to 0 like Theano's class Round3(UnaryScalarOp): def c_code(self, node, name, (x, ), (z, ), sub): return "%(z)s = round(%(x)s);" % locals() def grad(self, inputs, gout): (gz, ) = gout return gz, round3_scalar = Round3(same_out_nocomplex, name='round3') round3 = Elemwise(round3_scalar) def hard_sigmoid(x): return T.clip((x + 1.) / 2., 0, 1) # The neurons' activations binarization function # It behaves like the sign function during forward propagation # And like: # hard_tanh(x) = 2*hard_sigmoid(x)-1 # during back propagation def binary_tanh_unit(x): return 2. * round3(hard_sigmoid(x)) - 1.
def grad(self, inputs, gout): (gz, ) = gout return gz, class Floor3(UnaryScalarOp): def c_code(self, node, name, (x, ), (z, ), sub): return "%(z)s = floor(%(x)s+.5);" % locals() # train -.5 and .5 def grad(self, inputs, gout): (gz, ) = gout return gz, round3_scalar = Round3(same_out_nocomplex, name='round3') round3 = Elemwise(round3_scalar) floor3_scalar = Floor3(same_out_nocomplex, name='floor3') floor3 = Elemwise(floor3_scalar) def hard_sigmoid(x): return T.clip((x + 1.) / 2., 0, 1) # The neurons' activations binarization function # It behaves like the sign function during forward propagation # And like: # hard_tanh(x) = 2*hard_sigmoid(x)-1 # during back propagation
from theano.scalar.basic import UnaryScalarOp, same_out_nocomplex from theano.tensor.elemwise import Elemwise from itertools import izip class round_custom(UnaryScalarOp): def c_code(self, node, name, (x, ), (z, ), sub): return "%(z)s = round(%(x)s);" % locals() def grad(self, inputs, gout): (gz, ) = gout return gz, round_scalar = round_custom(same_out_nocomplex, name='round_var') round_var = Elemwise(round_scalar) def hard_sigmoid(x): return T.clip((x + 1.) / 2., 0, 1) def discrete_neuron_3states(x): #discrete activation with three states return T.cast( round_var(hard_sigmoid(2 * (x - 1)) + hard_sigmoid(2 * (x + 1)) - 1), theano.config.floatX) # This class extends the Lasagne DenseLayer to support Probabilistic Discretization of Weights class DenseLayer( lasagne.layers.DenseLayer
from theano.scalar.basic import UnaryScalarOp, same_out_nocomplex from theano.tensor.elemwise import Elemwise class Round3(UnaryScalarOp): def c_code(self, node, name, x, z, sub): return "%(z)s = round(%(x)s);" % locals() def grad(self, inputs, gout): (gz, ) = gout return gz, round3 = Elemwise(Round3(same_out_nocomplex, name='round3'))