def make_node(self, x, gz):
assert isinstance(x, Variable)
assert isinstance(gz, Variable)
gx = tensor(dtype=scal.upcast(gz.dtype, x.dtype),
broadcastable=x.broadcastable)
op = self
return Apply(op, [x, gz], [gx])
def make_node(self, *inputs):
inputs = map(as_tensor_variable, inputs)
if len(inputs) != 2:
raise TypeError(
"Wrong number of inputs for %s (got %i, expected 2)" %
self)
i_broadcastables = [input.type.broadcastable for input in inputs]
bx, by = i_broadcastables
if len(bx) == 0: # x is a scalar
bz = by
else:
if len(by) >= 2: # y is a matrix or tensor
bz = bx[:-1] + by[:-2] + by[-1:]
elif len(by) == 1: # y is vector
bz = bx[:-1]
else: # y is a scalar
bz = bx
i_dtypes = [input.type.dtype for input in inputs]
outputs = [tensor(scal.upcast(*i_dtypes), bz)]
return theano.Apply(self, inputs, outputs)
def normal(self, size, avg=0.0, std=1.0, ndim=None,
dtype=None, nstreams=None):
"""
:param size: Can be a list of integers or Theano variables (ex: the
shape of another Theano Variable)
:param dtype: The output data type. If dtype is not specified, it will
be inferred from the dtype of low and high, but will be at least as
precise as floatX.
:param nstreams: Number of streams.
"""
# We need an even number of ]0,1[ samples. Then we split them
# in two halves. First half becomes our U1's for Box-Muller,
# second half our U2's. See Wikipedia page:
# http://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform
avg = as_tensor_variable(avg)
std = as_tensor_variable(std)
if dtype is None:
dtype = scal.upcast(config.floatX, avg.dtype, std.dtype)
avg = cast(avg, dtype)
std = cast(std, dtype)
evened = False
constant = False
if isinstance(size, tuple) and all([isinstance(i, (numpy.integer, int)) for i in size]):
constant = True
# Force dtype because it defaults to float when size is empty
n_samples = numpy.prod(size, dtype='int64')
if n_samples % 2 == 1:
n_samples += 1
evened = True
else:
#if even, don't change, if odd, +1
n_samples = prod(size) + (prod(size) % 2)
flattened = self.uniform(size=(n_samples,), dtype=dtype,
nstreams=nstreams)
if constant:
U1 = flattened[:n_samples // 2]
U2 = flattened[n_samples // 2:]
else:
U1 = flattened[:prod(flattened.shape) // 2]
U2 = flattened[prod(flattened.shape) // 2:]
#normal_samples = zeros_like(flattened)
sqrt_ln_U1 = sqrt(-2.0 * log(U1))
# TypeError: 'TensorVariable' object does not support item assignment
# so this doesn't work...
#normal_samples[:n_samples/2] = sqrt_ln_U1 * cos(2.0*numpy.pi*U2)
#normal_samples[n_samples/2:] = sqrt_ln_U1 * sin(2.0*numpy.pi*U2)
# so trying this instead
first_half = sqrt_ln_U1 * cos(numpy.array(2.0 * numpy.pi, dtype=dtype) * U2)
second_half = sqrt_ln_U1 * sin(numpy.array(2.0 * numpy.pi, dtype=dtype) * U2)
normal_samples = join(0, first_half, second_half)
final_samples = None
if evened:
final_samples = normal_samples[:-1]
elif constant:
final_samples = normal_samples
else:
final_samples = normal_samples[:prod(size)]
if not size:
# Force the dtype to be int64, otherwise reshape complains
size = tensor.constant(size, dtype='int64')
final_samples = final_samples.reshape(size)
final_samples = avg + std * final_samples
assert final_samples.dtype == dtype
return final_samples
def normal(self,
size,
avg=0.0,
std=1.0,
ndim=None,
dtype=None,
nstreams=None):
"""
:param size: Can be a list of integers or Theano variables (ex: the
shape of another Theano Variable)
:param dtype: The output data type. If dtype is not specified, it will
be inferred from the dtype of low and high, but will be at least as
precise as floatX.
:param nstreams: Number of streams.
"""
# We need an even number of ]0,1[ samples. Then we split them
# in two halves. First half becomes our U1's for Box-Muller,
# second half our U2's. See Wikipedia page:
# http://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform
avg = as_tensor_variable(avg)
std = as_tensor_variable(std)
if dtype is None:
dtype = scal.upcast(config.floatX, avg.dtype, std.dtype)
avg = cast(avg, dtype)
std = cast(std, dtype)
evened = False
constant = False
if isinstance(size, tuple) and all(
[isinstance(i, (numpy.integer, int)) for i in size]):
constant = True
# Force dtype because it defaults to float when size is empty
n_samples = numpy.prod(size, dtype='int64')
if n_samples % 2 == 1:
n_samples += 1
evened = True
else:
#if even, don't change, if odd, +1
n_samples = prod(size) + (prod(size) % 2)
flattened = self.uniform(size=(n_samples, ),
dtype=dtype,
nstreams=nstreams)
if constant:
U1 = flattened[:n_samples // 2]
U2 = flattened[n_samples // 2:]
else:
U1 = flattened[:prod(flattened.shape) // 2]
U2 = flattened[prod(flattened.shape) // 2:]
#normal_samples = zeros_like(flattened)
sqrt_ln_U1 = sqrt(-2.0 * log(U1))
# TypeError: 'TensorVariable' object does not support item assignment
# so this doesn't work...
#normal_samples[:n_samples/2] = sqrt_ln_U1 * cos(2.0*numpy.pi*U2)
#normal_samples[n_samples/2:] = sqrt_ln_U1 * sin(2.0*numpy.pi*U2)
# so trying this instead
first_half = sqrt_ln_U1 * cos(
numpy.array(2.0 * numpy.pi, dtype=dtype) * U2)
second_half = sqrt_ln_U1 * sin(
numpy.array(2.0 * numpy.pi, dtype=dtype) * U2)
normal_samples = join(0, first_half, second_half)
final_samples = None
if evened:
final_samples = normal_samples[:-1]
elif constant:
final_samples = normal_samples
else:
final_samples = normal_samples[:prod(size)]
if not size:
# Force the dtype to be int64, otherwise reshape complains
size = tensor.constant(size, dtype='int64')
final_samples = final_samples.reshape(size)
final_samples = avg + std * final_samples
assert final_samples.dtype == dtype
return final_samples
def uniform(self, size, low=0.0, high=1.0, ndim=None, dtype=None,
nstreams=None):
"""
Sample a tensor of given size whose element from a uniform
distribution between low and high.
If the size argument is ambiguous on the number of dimensions,
ndim may be a plain integer to supplement the missing
information.
:param low: Lower bound of the interval on which values are sampled.
If the ``dtype`` arg is provided, ``low`` will be cast into dtype.
This bound is excluded.
:param high: Higher bound of the interval on which values are sampled.
If the ``dtype`` arg is provided, ``high`` will be cast into dtype.
This bound is excluded.
:param size: Can be a list of integer or Theano variable
(ex: the shape of other Theano Variable)
:param dtype: The output data type. If dtype is not specified, it will
be inferred from the dtype of low and high, but will be at least as
precise as floatX.
"""
low = as_tensor_variable(low)
high = as_tensor_variable(high)
if dtype is None:
dtype = scal.upcast(config.floatX, low.dtype, high.dtype)
low = cast(low, dtype=dtype)
high = cast(high, dtype=dtype)
if isinstance(size, tuple):
msg = "size must be a tuple of int or a Theano variable"
assert all([isinstance(i, (numpy.integer, int, Variable))
for i in size]), msg
if any([isinstance(i, (numpy.integer, int)) and i <= 0 for i in size]):
raise ValueError(
"The specified size contains a dimension with value <= 0",
size)
else:
if not (isinstance(size, Variable) and size.ndim == 1):
raise TypeError("size must be a tuple of int or a Theano "
"Variable with 1 dimension, got " + str(size) +
" of type " + str(type(size)))
if nstreams is None:
nstreams = self.n_streams(size)
if self.use_cuda and dtype == 'float32':
rstates = self.get_substream_rstates(nstreams)
rstates = rstates.flatten()
# HACK - we use fact that int32 and float32 have same size to
# sneak ints into the CudaNdarray type.
# these *SHOULD NEVER BE USED AS FLOATS*
tmp_float_buf = numpy.frombuffer(rstates.data, dtype='float32')
assert tmp_float_buf.shape == rstates.shape
assert (tmp_float_buf.view('int32') == rstates).all()
# transfer to device
node_rstate = float32_shared_constructor(tmp_float_buf)
assert isinstance(node_rstate.type, CudaNdarrayType)
# we can't use the normal mrg_uniform constructor + later
# optimization
# because of the tmp_float_buf hack above. There is
# currently no Theano node that will do a frombuffer
# reinterpretation.
u = self.pretty_return(node_rstate,
*GPU_mrg_uniform.new(node_rstate,
ndim, dtype, size))
else:
node_rstate = shared(self.get_substream_rstates(nstreams))
u = self.pretty_return(node_rstate,
*mrg_uniform.new(node_rstate,
ndim, dtype, size))
r = u * (high - low) + low
if u.type.broadcastable != r.type.broadcastable:
raise NotImplementedError(
'Increase the size to match the broadcasting pattern of '
'`low` and `high` arguments')
assert r.dtype == dtype
return r
def uniform(self,
size,
low=0.0,
high=1.0,
ndim=None,
dtype=None,
nstreams=None):
"""
Sample a tensor of given size whose element from a uniform
distribution between low and high.
If the size argument is ambiguous on the number of dimensions,
ndim may be a plain integer to supplement the missing
information.
:param low: Lower bound of the interval on which values are sampled.
If the ``dtype`` arg is provided, ``low`` will be cast into dtype.
:param high: Higher bound of the interval on which values are sampled.
If the ``dtype`` arg is provided, ``high`` will be cast into dtype.
:param size: Can be a list of integer or Theano variable
(ex: the shape of other Theano Variable)
:param dtype: The output data type. If dtype is not specified, it will
be inferred from the dtype of low and high, but will be at least as
precise as floatX.
"""
low = as_tensor_variable(low)
high = as_tensor_variable(high)
if dtype is None:
dtype = scal.upcast(config.floatX, low.dtype, high.dtype)
low = cast(low, dtype=dtype)
high = cast(high, dtype=dtype)
if isinstance(size, tuple):
msg = "size must be a tuple of int or a Theano variable"
assert all([
isinstance(i, (numpy.integer, int, Variable)) for i in size
]), msg
if any(
[isinstance(i, (numpy.integer, int)) and i <= 0
for i in size]):
raise ValueError(
"The specified size contains a dimension with value <= 0",
size)
else:
if not (isinstance(size, Variable) and size.ndim == 1):
raise TypeError("size must be a tuple of int or a Theano "
"Variable with 1 dimension, got " + str(size) +
" of type " + str(type(size)))
if nstreams is None:
nstreams = self.n_streams(size)
if self.use_cuda and dtype == 'float32':
rstates = self.get_substream_rstates(nstreams)
rstates = rstates.flatten()
# HACK - we use fact that int32 and float32 have same size to
# sneak ints into the CudaNdarray type.
# these *SHOULD NEVER BE USED AS FLOATS*
tmp_float_buf = numpy.frombuffer(rstates.data, dtype='float32')
assert tmp_float_buf.shape == rstates.shape
assert (tmp_float_buf.view('int32') == rstates).all()
# transfer to device
node_rstate = float32_shared_constructor(tmp_float_buf)
assert isinstance(node_rstate.type, CudaNdarrayType)
# we can't use the normal mrg_uniform constructor + later
# optimization
# because of the tmp_float_buf hack above. There is
# currently no Theano node that will do a frombuffer
# reinterpretation.
u = self.pretty_return(
node_rstate,
*GPU_mrg_uniform.new(node_rstate, ndim, dtype, size))
else:
node_rstate = shared(self.get_substream_rstates(nstreams))
u = self.pretty_return(
node_rstate, *mrg_uniform.new(node_rstate, ndim, dtype, size))
r = u * (high - low) + low
if u.type.broadcastable != r.type.broadcastable:
raise NotImplementedError(
'Increase the size to match the broadcasting pattern of '
'`low` and `high` arguments')
assert r.dtype == dtype
return r
def uniform(self,
size,
low=0.0,
high=1.0,
ndim=None,
dtype=None,
nstreams=None):
# TODO : need description for parameter 'size', 'ndim', 'nstreams'
"""
Sample a tensor of given size whose element from a uniform
distribution between low and high.
If the size argument is ambiguous on the number of dimensions,
ndim may be a plain integer to supplement the missing information.
Parameters
----------
low
Lower bound of the interval on which values are sampled.
If the ``dtype`` arg is provided, ``low`` will be cast into
dtype. This bound is excluded.
high
Higher bound of the interval on which values are sampled.
If the ``dtype`` arg is provided, ``high`` will be cast into
dtype. This bound is excluded.
size
Can be a list of integer or Theano variable (ex: the shape
of other Theano Variable).
dtype
The output data type. If dtype is not specified, it will be
inferred from the dtype of low and high, but will be at
least as precise as floatX.
"""
low = as_tensor_variable(low)
high = as_tensor_variable(high)
if dtype is None:
dtype = scal.upcast(config.floatX, low.dtype, high.dtype)
low = cast(low, dtype=dtype)
high = cast(high, dtype=dtype)
if isinstance(size, tuple):
msg = "size must be a tuple of int or a Theano variable"
assert all([
isinstance(i, (np.integer, integer_types, Variable))
for i in size
]), msg
if any([
isinstance(i, (np.integer, integer_types)) and i <= 0
for i in size
]):
raise ValueError(
"The specified size contains a dimension with value <= 0",
size)
else:
if not (isinstance(size, Variable) and size.ndim == 1):
raise TypeError("size must be a tuple of int or a Theano "
"Variable with 1 dimension, got " + str(size) +
" of type " + str(type(size)))
orig_nstreams = nstreams
if nstreams is None:
nstreams = self.n_streams(size)
rstates = self.get_substream_rstates(nstreams, dtype)
node_rstate = shared(rstates)
u = self.pretty_return(node_rstate,
*mrg_uniform.new(node_rstate, ndim, dtype,
size),
size=size,
nstreams=orig_nstreams)
# Add a reference to distinguish from other shared variables
node_rstate.tag.is_rng = True
r = u * (high - low) + low
if u.type.broadcastable != r.type.broadcastable:
raise NotImplementedError(
'Increase the size to match the broadcasting pattern of '
'`low` and `high` arguments')
assert r.dtype == dtype
return r
def uniform(self, size, low=0.0, high=1.0, ndim=None, dtype=None,
nstreams=None):
# TODO : need description for parameter 'size', 'ndim', 'nstreams'
"""
Sample a tensor of given size whose element from a uniform
distribution between low and high.
If the size argument is ambiguous on the number of dimensions,
ndim may be a plain integer to supplement the missing information.
Parameters
----------
low
Lower bound of the interval on which values are sampled.
If the ``dtype`` arg is provided, ``low`` will be cast into
dtype. This bound is excluded.
high
Higher bound of the interval on which values are sampled.
If the ``dtype`` arg is provided, ``high`` will be cast into
dtype. This bound is excluded.
size
Can be a list of integer or Theano variable (ex: the shape
of other Theano Variable).
dtype
The output data type. If dtype is not specified, it will be
inferred from the dtype of low and high, but will be at
least as precise as floatX.
"""
low = as_tensor_variable(low)
low = undefined_grad(low)
high = as_tensor_variable(high)
high = undefined_grad(high)
if dtype is None:
dtype = scal.upcast(config.floatX, low.dtype, high.dtype)
low = cast(low, dtype=dtype)
high = cast(high, dtype=dtype)
if isinstance(size, tuple):
msg = "size must be a tuple of int or a Theano variable"
assert all([isinstance(i, (np.integer, integer_types, Variable))
for i in size]), msg
if any([isinstance(i, (np.integer, integer_types)) and i <= 0
for i in size]):
raise ValueError(
"The specified size contains a dimension with value <= 0",
size)
else:
if not (isinstance(size, Variable) and size.ndim == 1):
raise TypeError("size must be a tuple of int or a Theano "
"Variable with 1 dimension, got " + str(size) +
" of type " + str(type(size)))
orig_nstreams = nstreams
if nstreams is None:
nstreams = self.n_streams(size)
rstates = self.get_substream_rstates(nstreams, dtype)
node_rstate = shared(rstates)
u = self.pretty_return(node_rstate,
*mrg_uniform.new(node_rstate,
ndim, dtype, size),
size=size, nstreams=orig_nstreams)
# Add a reference to distinguish from other shared variables
node_rstate.tag.is_rng = True
r = u * (high - low) + low
if u.type.broadcastable != r.type.broadcastable:
raise NotImplementedError(
'Increase the size to match the broadcasting pattern of '
'`low` and `high` arguments')
assert r.dtype == dtype
return r
def normal(self, size, avg=0.0, std=1.0, ndim=None, dtype=None,
nstreams=None, truncate=False, **kwargs):
"""
Sample a tensor of values from a normal distribution.
Parameters
----------
size : int_vector_like
Array dimensions for the output tensor.
avg : float_like, optional
The mean value for the truncated normal to sample from (defaults to 0.0).
std : float_like, optional
The standard deviation for the truncated normal to sample from (defaults to 1.0).
truncate : bool, optional
Truncates the normal distribution at 2 standard deviations if True (defaults to False).
When this flag is set, the standard deviation of the result will be less than the one specified.
ndim : int, optional
The number of dimensions for the output tensor (defaults to None).
This argument is necessary if the size argument is ambiguous on the number of dimensions.
dtype : str, optional
The data-type for the output tensor. If not specified,
the dtype is inferred from avg and std, but it is at least as precise as floatX.
kwargs
Other keyword arguments for random number generation (see uniform).
Returns
-------
samples : TensorVariable
A Theano tensor of samples randomly drawn from a normal distribution.
"""
size = _check_size(size)
avg = undefined_grad(as_tensor_variable(avg))
std = undefined_grad(as_tensor_variable(std))
if dtype is None:
dtype = scal.upcast(config.floatX, avg.dtype, std.dtype)
avg = tensor.cast(avg, dtype=dtype)
std = tensor.cast(std, dtype=dtype)
# generate even number of uniform samples
# Do manual constant folding to lower optiimizer work.
if isinstance(size, theano.Constant):
n_odd_samples = size.prod(dtype='int64')
else:
n_odd_samples = tensor.prod(size, dtype='int64')
n_even_samples = n_odd_samples + n_odd_samples % 2
uniform = self.uniform((n_even_samples, ), low=0., high=1.,
ndim=1, dtype=dtype, nstreams=nstreams, **kwargs)
# box-muller transform
u1 = uniform[:n_even_samples // 2]
u2 = uniform[n_even_samples // 2:]
r = tensor.sqrt(-2.0 * tensor.log(u1))
theta = np.array(2.0 * np.pi, dtype=dtype) * u2
cos_theta, sin_theta = tensor.cos(theta), tensor.sin(theta)
z0 = r * cos_theta
z1 = r * sin_theta
if truncate:
# use valid samples
to_fix0 = (z0 < -2.) | (z0 > 2.)
to_fix1 = (z1 < -2.) | (z1 > 2.)
z0_valid = z0[tensor.nonzero(~to_fix0)]
z1_valid = z1[tensor.nonzero(~to_fix1)]
# re-sample invalid samples
to_fix0 = tensor.nonzero(to_fix0)[0]
to_fix1 = tensor.nonzero(to_fix1)[0]
n_fix_samples = to_fix0.size + to_fix1.size
lower = tensor.constant(1. / np.e**2, dtype=dtype)
u_fix = self.uniform((n_fix_samples, ), low=lower, high=1.,
ndim=1, dtype=dtype, nstreams=nstreams, **kwargs)
r_fix = tensor.sqrt(-2. * tensor.log(u_fix))
z0_fixed = r_fix[:to_fix0.size] * cos_theta[to_fix0]
z1_fixed = r_fix[to_fix0.size:] * sin_theta[to_fix1]
# pack everything together to a useful result
norm_samples = tensor.join(0, z0_valid, z0_fixed, z1_valid, z1_fixed)
else:
norm_samples = tensor.join(0, z0, z1)
if isinstance(n_odd_samples, theano.Variable):
samples = norm_samples[:n_odd_samples]
elif n_odd_samples % 2 == 1:
samples = norm_samples[:-1]
else:
samples = norm_samples
samples = tensor.reshape(samples, newshape=size, ndim=ndim)
samples *= std
samples += avg
return samples
def normal(
self,
size,
avg=0.0,
std=1.0,
ndim=None,
dtype=None,
nstreams=None,
truncate=False,
**kwargs,
):
"""
Sample a tensor of values from a normal distribution.
Parameters
----------
size : int_vector_like
Array dimensions for the output tensor.
avg : float_like, optional
The mean value for the truncated normal to sample from (defaults to 0.0).
std : float_like, optional
The standard deviation for the truncated normal to sample from (defaults to 1.0).
truncate : bool, optional
Truncates the normal distribution at 2 standard deviations if True (defaults to False).
When this flag is set, the standard deviation of the result will be less than the one specified.
ndim : int, optional
The number of dimensions for the output tensor (defaults to None).
This argument is necessary if the size argument is ambiguous on the number of dimensions.
dtype : str, optional
The data-type for the output tensor. If not specified,
the dtype is inferred from avg and std, but it is at least as precise as floatX.
kwargs
Other keyword arguments for random number generation (see uniform).
Returns
-------
samples : TensorVariable
A Theano tensor of samples randomly drawn from a normal distribution.
"""
size = _check_size(size)
avg = undefined_grad(as_tensor_variable(avg))
std = undefined_grad(as_tensor_variable(std))
if dtype is None:
dtype = scal.upcast(config.floatX, avg.dtype, std.dtype)
avg = tensor.cast(avg, dtype=dtype)
std = tensor.cast(std, dtype=dtype)
# generate even number of uniform samples
# Do manual constant folding to lower optiimizer work.
if isinstance(size, theano.Constant):
n_odd_samples = size.prod(dtype="int64")
else:
n_odd_samples = tensor.prod(size, dtype="int64")
n_even_samples = n_odd_samples + n_odd_samples % 2
uniform = self.uniform(
(n_even_samples, ),
low=0.0,
high=1.0,
ndim=1,
dtype=dtype,
nstreams=nstreams,
**kwargs,
)
# box-muller transform
u1 = uniform[:n_even_samples // 2]
u2 = uniform[n_even_samples // 2:]
r = tensor.sqrt(-2.0 * tensor.log(u1))
theta = np.array(2.0 * np.pi, dtype=dtype) * u2
cos_theta, sin_theta = tensor.cos(theta), tensor.sin(theta)
z0 = r * cos_theta
z1 = r * sin_theta
if truncate:
# use valid samples
to_fix0 = (z0 < -2.0) | (z0 > 2.0)
to_fix1 = (z1 < -2.0) | (z1 > 2.0)
z0_valid = z0[tensor.nonzero(~to_fix0)]
z1_valid = z1[tensor.nonzero(~to_fix1)]
# re-sample invalid samples
to_fix0 = tensor.nonzero(to_fix0)[0]
to_fix1 = tensor.nonzero(to_fix1)[0]
n_fix_samples = to_fix0.size + to_fix1.size
lower = tensor.constant(1.0 / np.e**2, dtype=dtype)
u_fix = self.uniform(
(n_fix_samples, ),
low=lower,
high=1.0,
ndim=1,
dtype=dtype,
nstreams=nstreams,
**kwargs,
)
r_fix = tensor.sqrt(-2.0 * tensor.log(u_fix))
z0_fixed = r_fix[:to_fix0.size] * cos_theta[to_fix0]
z1_fixed = r_fix[to_fix0.size:] * sin_theta[to_fix1]
# pack everything together to a useful result
norm_samples = tensor.join(0, z0_valid, z0_fixed, z1_valid,
z1_fixed)
else:
norm_samples = tensor.join(0, z0, z1)
if isinstance(n_odd_samples, theano.Variable):
samples = norm_samples[:n_odd_samples]
elif n_odd_samples % 2 == 1:
samples = norm_samples[:-1]
else:
samples = norm_samples
samples = tensor.reshape(samples, newshape=size, ndim=ndim)
samples *= std
samples += avg
return samples
