def make_node(self, x, gz): assert isinstance(x, Variable) assert isinstance(gz, Variable) gx = tensor(dtype=scal.upcast(gz.dtype, x.dtype), broadcastable=x.broadcastable) op = self return Apply(op, [x, gz], [gx])
def make_node(self, *inputs): inputs = map(as_tensor_variable, inputs) if len(inputs) != 2: raise TypeError( "Wrong number of inputs for %s (got %i, expected 2)" % self) i_broadcastables = [input.type.broadcastable for input in inputs] bx, by = i_broadcastables if len(bx) == 0: # x is a scalar bz = by else: if len(by) >= 2: # y is a matrix or tensor bz = bx[:-1] + by[:-2] + by[-1:] elif len(by) == 1: # y is vector bz = bx[:-1] else: # y is a scalar bz = bx i_dtypes = [input.type.dtype for input in inputs] outputs = [tensor(scal.upcast(*i_dtypes), bz)] return theano.Apply(self, inputs, outputs)
def normal(self, size, avg=0.0, std=1.0, ndim=None, dtype=None, nstreams=None): """ :param size: Can be a list of integers or Theano variables (ex: the shape of another Theano Variable) :param dtype: The output data type. If dtype is not specified, it will be inferred from the dtype of low and high, but will be at least as precise as floatX. :param nstreams: Number of streams. """ # We need an even number of ]0,1[ samples. Then we split them # in two halves. First half becomes our U1's for Box-Muller, # second half our U2's. See Wikipedia page: # http://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform avg = as_tensor_variable(avg) std = as_tensor_variable(std) if dtype is None: dtype = scal.upcast(config.floatX, avg.dtype, std.dtype) avg = cast(avg, dtype) std = cast(std, dtype) evened = False constant = False if isinstance(size, tuple) and all([isinstance(i, (numpy.integer, int)) for i in size]): constant = True # Force dtype because it defaults to float when size is empty n_samples = numpy.prod(size, dtype='int64') if n_samples % 2 == 1: n_samples += 1 evened = True else: #if even, don't change, if odd, +1 n_samples = prod(size) + (prod(size) % 2) flattened = self.uniform(size=(n_samples,), dtype=dtype, nstreams=nstreams) if constant: U1 = flattened[:n_samples // 2] U2 = flattened[n_samples // 2:] else: U1 = flattened[:prod(flattened.shape) // 2] U2 = flattened[prod(flattened.shape) // 2:] #normal_samples = zeros_like(flattened) sqrt_ln_U1 = sqrt(-2.0 * log(U1)) # TypeError: 'TensorVariable' object does not support item assignment # so this doesn't work... #normal_samples[:n_samples/2] = sqrt_ln_U1 * cos(2.0*numpy.pi*U2) #normal_samples[n_samples/2:] = sqrt_ln_U1 * sin(2.0*numpy.pi*U2) # so trying this instead first_half = sqrt_ln_U1 * cos(numpy.array(2.0 * numpy.pi, dtype=dtype) * U2) second_half = sqrt_ln_U1 * sin(numpy.array(2.0 * numpy.pi, dtype=dtype) * U2) normal_samples = join(0, first_half, second_half) final_samples = None if evened: final_samples = normal_samples[:-1] elif constant: final_samples = normal_samples else: final_samples = normal_samples[:prod(size)] if not size: # Force the dtype to be int64, otherwise reshape complains size = tensor.constant(size, dtype='int64') final_samples = final_samples.reshape(size) final_samples = avg + std * final_samples assert final_samples.dtype == dtype return final_samples
def normal(self, size, avg=0.0, std=1.0, ndim=None, dtype=None, nstreams=None): """ :param size: Can be a list of integers or Theano variables (ex: the shape of another Theano Variable) :param dtype: The output data type. If dtype is not specified, it will be inferred from the dtype of low and high, but will be at least as precise as floatX. :param nstreams: Number of streams. """ # We need an even number of ]0,1[ samples. Then we split them # in two halves. First half becomes our U1's for Box-Muller, # second half our U2's. See Wikipedia page: # http://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform avg = as_tensor_variable(avg) std = as_tensor_variable(std) if dtype is None: dtype = scal.upcast(config.floatX, avg.dtype, std.dtype) avg = cast(avg, dtype) std = cast(std, dtype) evened = False constant = False if isinstance(size, tuple) and all( [isinstance(i, (numpy.integer, int)) for i in size]): constant = True # Force dtype because it defaults to float when size is empty n_samples = numpy.prod(size, dtype='int64') if n_samples % 2 == 1: n_samples += 1 evened = True else: #if even, don't change, if odd, +1 n_samples = prod(size) + (prod(size) % 2) flattened = self.uniform(size=(n_samples, ), dtype=dtype, nstreams=nstreams) if constant: U1 = flattened[:n_samples // 2] U2 = flattened[n_samples // 2:] else: U1 = flattened[:prod(flattened.shape) // 2] U2 = flattened[prod(flattened.shape) // 2:] #normal_samples = zeros_like(flattened) sqrt_ln_U1 = sqrt(-2.0 * log(U1)) # TypeError: 'TensorVariable' object does not support item assignment # so this doesn't work... #normal_samples[:n_samples/2] = sqrt_ln_U1 * cos(2.0*numpy.pi*U2) #normal_samples[n_samples/2:] = sqrt_ln_U1 * sin(2.0*numpy.pi*U2) # so trying this instead first_half = sqrt_ln_U1 * cos( numpy.array(2.0 * numpy.pi, dtype=dtype) * U2) second_half = sqrt_ln_U1 * sin( numpy.array(2.0 * numpy.pi, dtype=dtype) * U2) normal_samples = join(0, first_half, second_half) final_samples = None if evened: final_samples = normal_samples[:-1] elif constant: final_samples = normal_samples else: final_samples = normal_samples[:prod(size)] if not size: # Force the dtype to be int64, otherwise reshape complains size = tensor.constant(size, dtype='int64') final_samples = final_samples.reshape(size) final_samples = avg + std * final_samples assert final_samples.dtype == dtype return final_samples
def uniform(self, size, low=0.0, high=1.0, ndim=None, dtype=None, nstreams=None): """ Sample a tensor of given size whose element from a uniform distribution between low and high. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. :param low: Lower bound of the interval on which values are sampled. If the ``dtype`` arg is provided, ``low`` will be cast into dtype. This bound is excluded. :param high: Higher bound of the interval on which values are sampled. If the ``dtype`` arg is provided, ``high`` will be cast into dtype. This bound is excluded. :param size: Can be a list of integer or Theano variable (ex: the shape of other Theano Variable) :param dtype: The output data type. If dtype is not specified, it will be inferred from the dtype of low and high, but will be at least as precise as floatX. """ low = as_tensor_variable(low) high = as_tensor_variable(high) if dtype is None: dtype = scal.upcast(config.floatX, low.dtype, high.dtype) low = cast(low, dtype=dtype) high = cast(high, dtype=dtype) if isinstance(size, tuple): msg = "size must be a tuple of int or a Theano variable" assert all([isinstance(i, (numpy.integer, int, Variable)) for i in size]), msg if any([isinstance(i, (numpy.integer, int)) and i <= 0 for i in size]): raise ValueError( "The specified size contains a dimension with value <= 0", size) else: if not (isinstance(size, Variable) and size.ndim == 1): raise TypeError("size must be a tuple of int or a Theano " "Variable with 1 dimension, got " + str(size) + " of type " + str(type(size))) if nstreams is None: nstreams = self.n_streams(size) if self.use_cuda and dtype == 'float32': rstates = self.get_substream_rstates(nstreams) rstates = rstates.flatten() # HACK - we use fact that int32 and float32 have same size to # sneak ints into the CudaNdarray type. # these *SHOULD NEVER BE USED AS FLOATS* tmp_float_buf = numpy.frombuffer(rstates.data, dtype='float32') assert tmp_float_buf.shape == rstates.shape assert (tmp_float_buf.view('int32') == rstates).all() # transfer to device node_rstate = float32_shared_constructor(tmp_float_buf) assert isinstance(node_rstate.type, CudaNdarrayType) # we can't use the normal mrg_uniform constructor + later # optimization # because of the tmp_float_buf hack above. There is # currently no Theano node that will do a frombuffer # reinterpretation. u = self.pretty_return(node_rstate, *GPU_mrg_uniform.new(node_rstate, ndim, dtype, size)) else: node_rstate = shared(self.get_substream_rstates(nstreams)) u = self.pretty_return(node_rstate, *mrg_uniform.new(node_rstate, ndim, dtype, size)) r = u * (high - low) + low if u.type.broadcastable != r.type.broadcastable: raise NotImplementedError( 'Increase the size to match the broadcasting pattern of ' '`low` and `high` arguments') assert r.dtype == dtype return r
def uniform(self, size, low=0.0, high=1.0, ndim=None, dtype=None, nstreams=None): """ Sample a tensor of given size whose element from a uniform distribution between low and high. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. :param low: Lower bound of the interval on which values are sampled. If the ``dtype`` arg is provided, ``low`` will be cast into dtype. :param high: Higher bound of the interval on which values are sampled. If the ``dtype`` arg is provided, ``high`` will be cast into dtype. :param size: Can be a list of integer or Theano variable (ex: the shape of other Theano Variable) :param dtype: The output data type. If dtype is not specified, it will be inferred from the dtype of low and high, but will be at least as precise as floatX. """ low = as_tensor_variable(low) high = as_tensor_variable(high) if dtype is None: dtype = scal.upcast(config.floatX, low.dtype, high.dtype) low = cast(low, dtype=dtype) high = cast(high, dtype=dtype) if isinstance(size, tuple): msg = "size must be a tuple of int or a Theano variable" assert all([ isinstance(i, (numpy.integer, int, Variable)) for i in size ]), msg if any( [isinstance(i, (numpy.integer, int)) and i <= 0 for i in size]): raise ValueError( "The specified size contains a dimension with value <= 0", size) else: if not (isinstance(size, Variable) and size.ndim == 1): raise TypeError("size must be a tuple of int or a Theano " "Variable with 1 dimension, got " + str(size) + " of type " + str(type(size))) if nstreams is None: nstreams = self.n_streams(size) if self.use_cuda and dtype == 'float32': rstates = self.get_substream_rstates(nstreams) rstates = rstates.flatten() # HACK - we use fact that int32 and float32 have same size to # sneak ints into the CudaNdarray type. # these *SHOULD NEVER BE USED AS FLOATS* tmp_float_buf = numpy.frombuffer(rstates.data, dtype='float32') assert tmp_float_buf.shape == rstates.shape assert (tmp_float_buf.view('int32') == rstates).all() # transfer to device node_rstate = float32_shared_constructor(tmp_float_buf) assert isinstance(node_rstate.type, CudaNdarrayType) # we can't use the normal mrg_uniform constructor + later # optimization # because of the tmp_float_buf hack above. There is # currently no Theano node that will do a frombuffer # reinterpretation. u = self.pretty_return( node_rstate, *GPU_mrg_uniform.new(node_rstate, ndim, dtype, size)) else: node_rstate = shared(self.get_substream_rstates(nstreams)) u = self.pretty_return( node_rstate, *mrg_uniform.new(node_rstate, ndim, dtype, size)) r = u * (high - low) + low if u.type.broadcastable != r.type.broadcastable: raise NotImplementedError( 'Increase the size to match the broadcasting pattern of ' '`low` and `high` arguments') assert r.dtype == dtype return r
def uniform(self, size, low=0.0, high=1.0, ndim=None, dtype=None, nstreams=None): # TODO : need description for parameter 'size', 'ndim', 'nstreams' """ Sample a tensor of given size whose element from a uniform distribution between low and high. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. Parameters ---------- low Lower bound of the interval on which values are sampled. If the ``dtype`` arg is provided, ``low`` will be cast into dtype. This bound is excluded. high Higher bound of the interval on which values are sampled. If the ``dtype`` arg is provided, ``high`` will be cast into dtype. This bound is excluded. size Can be a list of integer or Theano variable (ex: the shape of other Theano Variable). dtype The output data type. If dtype is not specified, it will be inferred from the dtype of low and high, but will be at least as precise as floatX. """ low = as_tensor_variable(low) high = as_tensor_variable(high) if dtype is None: dtype = scal.upcast(config.floatX, low.dtype, high.dtype) low = cast(low, dtype=dtype) high = cast(high, dtype=dtype) if isinstance(size, tuple): msg = "size must be a tuple of int or a Theano variable" assert all([ isinstance(i, (np.integer, integer_types, Variable)) for i in size ]), msg if any([ isinstance(i, (np.integer, integer_types)) and i <= 0 for i in size ]): raise ValueError( "The specified size contains a dimension with value <= 0", size) else: if not (isinstance(size, Variable) and size.ndim == 1): raise TypeError("size must be a tuple of int or a Theano " "Variable with 1 dimension, got " + str(size) + " of type " + str(type(size))) orig_nstreams = nstreams if nstreams is None: nstreams = self.n_streams(size) rstates = self.get_substream_rstates(nstreams, dtype) node_rstate = shared(rstates) u = self.pretty_return(node_rstate, *mrg_uniform.new(node_rstate, ndim, dtype, size), size=size, nstreams=orig_nstreams) # Add a reference to distinguish from other shared variables node_rstate.tag.is_rng = True r = u * (high - low) + low if u.type.broadcastable != r.type.broadcastable: raise NotImplementedError( 'Increase the size to match the broadcasting pattern of ' '`low` and `high` arguments') assert r.dtype == dtype return r
def uniform(self, size, low=0.0, high=1.0, ndim=None, dtype=None, nstreams=None): # TODO : need description for parameter 'size', 'ndim', 'nstreams' """ Sample a tensor of given size whose element from a uniform distribution between low and high. If the size argument is ambiguous on the number of dimensions, ndim may be a plain integer to supplement the missing information. Parameters ---------- low Lower bound of the interval on which values are sampled. If the ``dtype`` arg is provided, ``low`` will be cast into dtype. This bound is excluded. high Higher bound of the interval on which values are sampled. If the ``dtype`` arg is provided, ``high`` will be cast into dtype. This bound is excluded. size Can be a list of integer or Theano variable (ex: the shape of other Theano Variable). dtype The output data type. If dtype is not specified, it will be inferred from the dtype of low and high, but will be at least as precise as floatX. """ low = as_tensor_variable(low) low = undefined_grad(low) high = as_tensor_variable(high) high = undefined_grad(high) if dtype is None: dtype = scal.upcast(config.floatX, low.dtype, high.dtype) low = cast(low, dtype=dtype) high = cast(high, dtype=dtype) if isinstance(size, tuple): msg = "size must be a tuple of int or a Theano variable" assert all([isinstance(i, (np.integer, integer_types, Variable)) for i in size]), msg if any([isinstance(i, (np.integer, integer_types)) and i <= 0 for i in size]): raise ValueError( "The specified size contains a dimension with value <= 0", size) else: if not (isinstance(size, Variable) and size.ndim == 1): raise TypeError("size must be a tuple of int or a Theano " "Variable with 1 dimension, got " + str(size) + " of type " + str(type(size))) orig_nstreams = nstreams if nstreams is None: nstreams = self.n_streams(size) rstates = self.get_substream_rstates(nstreams, dtype) node_rstate = shared(rstates) u = self.pretty_return(node_rstate, *mrg_uniform.new(node_rstate, ndim, dtype, size), size=size, nstreams=orig_nstreams) # Add a reference to distinguish from other shared variables node_rstate.tag.is_rng = True r = u * (high - low) + low if u.type.broadcastable != r.type.broadcastable: raise NotImplementedError( 'Increase the size to match the broadcasting pattern of ' '`low` and `high` arguments') assert r.dtype == dtype return r
def normal(self, size, avg=0.0, std=1.0, ndim=None, dtype=None, nstreams=None, truncate=False, **kwargs): """ Sample a tensor of values from a normal distribution. Parameters ---------- size : int_vector_like Array dimensions for the output tensor. avg : float_like, optional The mean value for the truncated normal to sample from (defaults to 0.0). std : float_like, optional The standard deviation for the truncated normal to sample from (defaults to 1.0). truncate : bool, optional Truncates the normal distribution at 2 standard deviations if True (defaults to False). When this flag is set, the standard deviation of the result will be less than the one specified. ndim : int, optional The number of dimensions for the output tensor (defaults to None). This argument is necessary if the size argument is ambiguous on the number of dimensions. dtype : str, optional The data-type for the output tensor. If not specified, the dtype is inferred from avg and std, but it is at least as precise as floatX. kwargs Other keyword arguments for random number generation (see uniform). Returns ------- samples : TensorVariable A Theano tensor of samples randomly drawn from a normal distribution. """ size = _check_size(size) avg = undefined_grad(as_tensor_variable(avg)) std = undefined_grad(as_tensor_variable(std)) if dtype is None: dtype = scal.upcast(config.floatX, avg.dtype, std.dtype) avg = tensor.cast(avg, dtype=dtype) std = tensor.cast(std, dtype=dtype) # generate even number of uniform samples # Do manual constant folding to lower optiimizer work. if isinstance(size, theano.Constant): n_odd_samples = size.prod(dtype='int64') else: n_odd_samples = tensor.prod(size, dtype='int64') n_even_samples = n_odd_samples + n_odd_samples % 2 uniform = self.uniform((n_even_samples, ), low=0., high=1., ndim=1, dtype=dtype, nstreams=nstreams, **kwargs) # box-muller transform u1 = uniform[:n_even_samples // 2] u2 = uniform[n_even_samples // 2:] r = tensor.sqrt(-2.0 * tensor.log(u1)) theta = np.array(2.0 * np.pi, dtype=dtype) * u2 cos_theta, sin_theta = tensor.cos(theta), tensor.sin(theta) z0 = r * cos_theta z1 = r * sin_theta if truncate: # use valid samples to_fix0 = (z0 < -2.) | (z0 > 2.) to_fix1 = (z1 < -2.) | (z1 > 2.) z0_valid = z0[tensor.nonzero(~to_fix0)] z1_valid = z1[tensor.nonzero(~to_fix1)] # re-sample invalid samples to_fix0 = tensor.nonzero(to_fix0)[0] to_fix1 = tensor.nonzero(to_fix1)[0] n_fix_samples = to_fix0.size + to_fix1.size lower = tensor.constant(1. / np.e**2, dtype=dtype) u_fix = self.uniform((n_fix_samples, ), low=lower, high=1., ndim=1, dtype=dtype, nstreams=nstreams, **kwargs) r_fix = tensor.sqrt(-2. * tensor.log(u_fix)) z0_fixed = r_fix[:to_fix0.size] * cos_theta[to_fix0] z1_fixed = r_fix[to_fix0.size:] * sin_theta[to_fix1] # pack everything together to a useful result norm_samples = tensor.join(0, z0_valid, z0_fixed, z1_valid, z1_fixed) else: norm_samples = tensor.join(0, z0, z1) if isinstance(n_odd_samples, theano.Variable): samples = norm_samples[:n_odd_samples] elif n_odd_samples % 2 == 1: samples = norm_samples[:-1] else: samples = norm_samples samples = tensor.reshape(samples, newshape=size, ndim=ndim) samples *= std samples += avg return samples
def normal( self, size, avg=0.0, std=1.0, ndim=None, dtype=None, nstreams=None, truncate=False, **kwargs, ): """ Sample a tensor of values from a normal distribution. Parameters ---------- size : int_vector_like Array dimensions for the output tensor. avg : float_like, optional The mean value for the truncated normal to sample from (defaults to 0.0). std : float_like, optional The standard deviation for the truncated normal to sample from (defaults to 1.0). truncate : bool, optional Truncates the normal distribution at 2 standard deviations if True (defaults to False). When this flag is set, the standard deviation of the result will be less than the one specified. ndim : int, optional The number of dimensions for the output tensor (defaults to None). This argument is necessary if the size argument is ambiguous on the number of dimensions. dtype : str, optional The data-type for the output tensor. If not specified, the dtype is inferred from avg and std, but it is at least as precise as floatX. kwargs Other keyword arguments for random number generation (see uniform). Returns ------- samples : TensorVariable A Theano tensor of samples randomly drawn from a normal distribution. """ size = _check_size(size) avg = undefined_grad(as_tensor_variable(avg)) std = undefined_grad(as_tensor_variable(std)) if dtype is None: dtype = scal.upcast(config.floatX, avg.dtype, std.dtype) avg = tensor.cast(avg, dtype=dtype) std = tensor.cast(std, dtype=dtype) # generate even number of uniform samples # Do manual constant folding to lower optiimizer work. if isinstance(size, theano.Constant): n_odd_samples = size.prod(dtype="int64") else: n_odd_samples = tensor.prod(size, dtype="int64") n_even_samples = n_odd_samples + n_odd_samples % 2 uniform = self.uniform( (n_even_samples, ), low=0.0, high=1.0, ndim=1, dtype=dtype, nstreams=nstreams, **kwargs, ) # box-muller transform u1 = uniform[:n_even_samples // 2] u2 = uniform[n_even_samples // 2:] r = tensor.sqrt(-2.0 * tensor.log(u1)) theta = np.array(2.0 * np.pi, dtype=dtype) * u2 cos_theta, sin_theta = tensor.cos(theta), tensor.sin(theta) z0 = r * cos_theta z1 = r * sin_theta if truncate: # use valid samples to_fix0 = (z0 < -2.0) | (z0 > 2.0) to_fix1 = (z1 < -2.0) | (z1 > 2.0) z0_valid = z0[tensor.nonzero(~to_fix0)] z1_valid = z1[tensor.nonzero(~to_fix1)] # re-sample invalid samples to_fix0 = tensor.nonzero(to_fix0)[0] to_fix1 = tensor.nonzero(to_fix1)[0] n_fix_samples = to_fix0.size + to_fix1.size lower = tensor.constant(1.0 / np.e**2, dtype=dtype) u_fix = self.uniform( (n_fix_samples, ), low=lower, high=1.0, ndim=1, dtype=dtype, nstreams=nstreams, **kwargs, ) r_fix = tensor.sqrt(-2.0 * tensor.log(u_fix)) z0_fixed = r_fix[:to_fix0.size] * cos_theta[to_fix0] z1_fixed = r_fix[to_fix0.size:] * sin_theta[to_fix1] # pack everything together to a useful result norm_samples = tensor.join(0, z0_valid, z0_fixed, z1_valid, z1_fixed) else: norm_samples = tensor.join(0, z0, z1) if isinstance(n_odd_samples, theano.Variable): samples = norm_samples[:n_odd_samples] elif n_odd_samples % 2 == 1: samples = norm_samples[:-1] else: samples = norm_samples samples = tensor.reshape(samples, newshape=size, ndim=ndim) samples *= std samples += avg return samples