def __init__(self, axis, value_ndims, **kwargs):
"""
Construct a new :class:`FeatureMappingFlow`.
Args:
axis (int or Iterable[int]): The feature axis/axes, on which to
apply the transformation.
value_ndims (int): Number of value dimensions in both `x` and `y`.
`x.ndims - value_ndims == log_det.ndims` and
`y.ndims - value_ndims == log_det.ndims`.
\\**kwargs: Other named arguments passed to :class:`BaseFlow`.
"""
# check the arguments
if is_integer(axis):
axis = int(axis)
else:
axis = tuple(int(a) for a in axis)
if not axis:
raise ValueError('`axis` must not be empty.')
if 'x_value_ndims' in kwargs or 'y_value_ndims' in kwargs:
raise ValueError('Specifying `x_value_ndims` or `y_value_ndims` '
'for a `FeatureMappingFlow` is not allowed.')
value_ndims = int(value_ndims)
# construct the layer
super(FeatureMappingFlow, self).__init__(x_value_ndims=value_ndims,
y_value_ndims=value_ndims,
**kwargs)
self._axis = axis
def __init__(self,
hidden_net_p_x_z,
hidden_net_q_z_x,
x_dims,
z_dims,
std_epsilon=1e-4,
name=None,
scope=None):
if not is_integer(x_dims) or x_dims <= 0:
raise ValueError('`x_dims`必须为正整数')
if not is_integer(z_dims) or z_dims <= 0:
raise ValueError('`z_dims`必须为正整数')
super(Donut, self).__init__(name=name, scope=scope)
with reopen_variable_scope(self.variable_scope):
# 基于VAE构造
self._vae = VAE(
# p(z)：均值和标准差都为z维数量大小的全零数组的一元正态分布
p_z=Normal(mean=tf.zeros([z_dims]), std=tf.ones([z_dims])),
# p(x|h(z))：一元正态分布
p_x_given_z=Normal,
# q(z|h(x))：一元正态分布
q_z_given_x=Normal,
# p(x|h(z))的隐藏网络：mean、std，由p(x|z)隐藏网络输入获得
h_for_p_x=Lambda(partial(wrap_params_net,
h_for_dist=hidden_net_p_x_z,
mean_layer=partial(tf.layers.dense,
units=x_dims,
name='x_mean'),
std_layer=partial(softplus_std,
units=x_dims,
epsilon=std_epsilon,
name='x_std')),
name='p_x_given_z'),
# q(z|h(x))的隐藏网络：mean、std，由q(z|x)隐藏网络输入获得
h_for_q_z=Lambda(partial(wrap_params_net,
h_for_dist=hidden_net_q_z_x,
mean_layer=partial(tf.layers.dense,
units=z_dims,
name='z_mean'),
std_layer=partial(softplus_std,
units=z_dims,
epsilon=std_epsilon,
name='z_std')),
name='q_z_given_x'))
self._x_dims = x_dims
self._z_dims = z_dims
def validate_size_tuple(n, s):
if is_integer(s):
# Do not change a single integer into a tuple!
# This is because we do not know the dimensionality of the
# convolution operation here, so we cannot build the size
# tuple with correct number of elements from the integer notation.
return int(s)
return validate_int_tuple_arg(n, s)
def __init__(self,
h_for_p_x,
h_for_q_z,
x_dims,
z_dims,
std_epsilon=1e-4,
name=None,
scope=None):
if not is_integer(x_dims) or x_dims <= 0:
raise ValueError('`x_dims` must be a positive integer')
if not is_integer(z_dims) or z_dims <= 0:
raise ValueError('`z_dims` must be a positive integer')
super(Donut, self).__init__(name=name, scope=scope)
with reopen_variable_scope(self.variable_scope):
self._vae = VAE(
p_z=Normal(mean=tf.zeros([z_dims]), std=tf.ones([z_dims])),
p_x_given_z=Normal,
q_z_given_x=Normal,
h_for_p_x=Sequential([
h_for_p_x,
DictMapper(
{
'mean':
K.layers.Dense(x_dims),
'std':
lambda x: (std_epsilon + K.layers.Dense(
x_dims, activation=tf.nn.softplus)(x))
},
name='p_x_given_z')
]),
h_for_q_z=Sequential([
h_for_q_z,
DictMapper(
{
'mean':
K.layers.Dense(z_dims),
'std':
lambda z: (std_epsilon + K.layers.Dense(
z_dims, activation=tf.nn.softplus)(z))
},
name='q_z_given_x')
]),
)
self._x_dims = x_dims
self._z_dims = z_dims
def test_is_integer(self):
if six.PY2:
self.assertTrue(is_integer(long(1)))
self.assertTrue(is_integer(int(1)))
for dtype in [np.int, np.int8, np.int16, np.int32, np.int64,
np.uint, np.uint8, np.uint16, np.uint32, np.uint64]:
v = np.asarray([1], dtype=dtype)[0]
self.assertTrue(
is_integer(v),
msg='%r should be interpreted as integer.' % (v,)
)
self.assertFalse(is_integer(np.asarray(0, dtype=np.int)))
for v in [float(1.0), '', object(), None, True, (), {}, []]:
self.assertFalse(
is_integer(v),
msg='%r should not be interpreted as integer.' % (v,)
)
def _build_input_spec(self, input):
super(FeatureMappingFlow, self)._build_input_spec(input)
dtype = input.dtype.base_dtype
shape = get_static_shape(input)
# These facts should have been checked in `BaseFlow.build`.
assert (shape is not None)
assert (len(shape) >= self.value_ndims)
# validate the feature axis, ensure it is covered by `value_ndims`.
axis = self._axis
axis_is_int = is_integer(axis)
if axis_is_int:
axis = [axis]
else:
axis = list(axis)
for i, a in enumerate(axis):
if a < 0:
a += len(shape)
if a < 0 or a < len(shape) - self.value_ndims:
raise ValueError('`axis` out of range, or not covered by '
'`value_ndims`: axis {}, value_ndims {}, '
'input {}'.format(self._axis,
self.value_ndims, input))
if shape[a] is None:
raise ValueError('The feature axis of `input` is not '
'deterministic: input {}, axis {}'.format(
input, self._axis))
# Store the negative axis, such that when new inputs can have more
# dimensions than this `input`, the axis can still be correctly
# resolved.
axis[i] = a - len(shape)
if axis_is_int:
assert (len(axis) == 1)
self._axis = axis[0]
else:
axis_len = len(axis)
axis = tuple(sorted(set(axis)))
if len(axis) != axis_len:
raise ValueError(
'Duplicated elements after resolving negative '
'`axis` with respect to the `input`: '
'input {}, axis {}'.format(input, self._axis))
self._axis = tuple(axis)
# re-build the input spec
batch_ndims = int(self.require_batch_dims)
shape_spec = ['...'] + ['?'] * (self.value_ndims + batch_ndims)
for a in axis:
shape_spec[a] = shape[a]
self._y_input_spec = self._x_input_spec = InputSpec(shape=shape_spec,
dtype=dtype)
self._x_input_spec.validate('input', input)
def __init__(self, h_for_p_x, h_for_q_z, x_dims, z_dims, std_epsilon=1e-4,
name=None, scope=None) -> object:
if not is_integer(x_dims) or x_dims <= 0:
raise ValueError('`x_dims` must be a positive integer')
if not is_integer(z_dims) or z_dims <= 0:
raise ValueError('`z_dims` must be a positive integer')
super(Donut, self).__init__(name=name, scope=scope)
with reopen_variable_scope(self.variable_scope):
self._vae = VAE(
p_z=Normal(mean=tf.zeros([z_dims]), std=tf.ones([z_dims])),
p_x_given_z=Normal,
q_z_given_x=Normal,
h_for_p_x=Lambda(
partial(
wrap_params_net,
h_for_dist=h_for_p_x,
mean_layer=partial(
tf.layers.dense, units=x_dims, name='x_mean'
),
std_layer=partial(
softplus_std, units=x_dims, epsilon=std_epsilon,
name='x_std'
)
),
name='p_x_given_z'
),
h_for_q_z=Lambda(
partial(
wrap_params_net,
h_for_dist=h_for_q_z,
mean_layer=partial(
tf.layers.dense, units=z_dims, name='z_mean'
),
std_layer=partial(
softplus_std, units=z_dims, epsilon=std_epsilon,
name='z_std'
)
),
name='q_z_given_x'
)
)
self._x_dims = x_dims
self._z_dims = z_dims
def log_prob(self, x, group_event_ndims=None, name=None):
"""Compute the log-probability of `x` against the distribution.
If `group_event_ndims` is configured, then the likelihoods of
one group of events will be summed together.
Parameters
----------
x : tf.Tensor
The samples to be tested.
group_event_ndims : int | tf.Tensor
If specified, will override the attribute `group_event_ndims`
of this distribution object.
name : str
Optional name of this operation.
Returns
-------
tf.Tensor
The log-probability of `x`.
"""
with tf.name_scope(name, default_name='log_prob'):
# determine the number of group event dimensions
if group_event_ndims is None:
group_event_ndims = self.group_event_ndims
# compute the log-likelihood
log_prob = self._log_prob(x)
log_prob_shape = log_prob.get_shape()
try:
tf.broadcast_static_shape(log_prob_shape,
self.static_batch_shape)
except ValueError:
raise RuntimeError(
'The shape of computed log-prob does not match '
'`batch_shape`, which could be a bug in the distribution '
'implementation. (%r vs %r)' %
(log_prob_shape, self.static_batch_shape))
# reduce the dimensions of group event
def f(ndims):
return tf.reduce_sum(log_prob, axis=tf.range(-ndims, 0))
if group_event_ndims is not None:
if is_integer(group_event_ndims):
if group_event_ndims > 0:
log_prob = f(group_event_ndims)
else:
group_event_ndims = tf.convert_to_tensor(group_event_ndims,
dtype=tf.int32)
return tf.cond(group_event_ndims > 0,
lambda: f(group_event_ndims),
lambda: log_prob)
return log_prob
def pool2d_ans(pool_fn, input, pool_size, padding, strides):
"""Produce the expected answer of ?_pool2d."""
strides = (strides, ) * 2 if is_integer(strides) else tuple(strides)
strides = (1, ) + strides + (1, )
ksize = (pool_size, ) * 2 if is_integer(pool_size) else tuple(
pool_size)
ksize = (1, ) + ksize + (1, )
session = tf.get_default_session()
input, s1, s2 = flatten_to_ndims(input, 4)
padding = padding.upper()
output = pool_fn(
value=input,
ksize=ksize,
strides=strides,
padding=padding,
data_format='NHWC',
)
output = unflatten_from_ndims(output, s1, s2)
output = session.run(output)
return output
def validate_strides_or_kernel_size(arg_name, arg_value):
"""
Validate the `strides` or `filter` arg, to ensure it is a tuple of
two integers.
Args:
arg_name (str): The name of the argument, for formatting error.
arg_value: The value of the argument.
Returns:
(int, int): The validated argument.
"""
if not is_integer(arg_value) and (not isinstance(arg_value, tuple)
or len(arg_value) != 2
or not is_integer(arg_value[0])
or not is_integer(arg_value[1])):
raise TypeError(
'`{}` must be a int or a tuple (int, int).'.format(arg_name))
if not isinstance(arg_value, tuple):
arg_value = (arg_value, arg_value)
arg_value = tuple(int(v) for v in arg_value)
return arg_value
def test_is_float(self):
float_types = [float, np.float, np.float16, np.float32, np.float64]
for extra_type in ['float8', 'float128', 'float256']:
if hasattr(np, extra_type):
float_types.append(getattr(np, extra_type))
for dtype in float_types:
v = np.asarray([1], dtype=dtype)[0]
self.assertTrue(
is_float(v),
msg='{!r} should be interpreted as float'.format(v))
self.assertFalse(is_integer(np.asarray(0., dtype=np.float32)))
for v in [int(1), '', object(), None, True, (), {}, []]:
self.assertFalse(
is_float(v),
msg='{!r} should not be interpreted as float'.format(v))
def conv2d_ans(input,
padding,
kernel,
bias,
strides,
dilations,
activation_fn=None,
normalizer_fn=None,
gated=False,
gate_sigmoid_bias=2.):
"""Produce the expected answer of conv2d."""
strides = (strides, ) * 2 if is_integer(strides) else tuple(strides)
strides = (1, ) + strides + (1, )
session = tf.get_default_session()
input, s1, s2 = flatten_to_ndims(input, 4)
padding = padding.upper()
if dilations > 1:
assert (not any(i > 1 for i in strides))
output = tf.nn.atrous_conv2d(value=input,
filters=kernel,
rate=dilations,
padding=padding)
else:
output = tf.nn.conv2d(input=input,
filter=kernel,
strides=strides,
padding=padding,
data_format='NHWC',
dilations=[1] * 4)
if bias is not None:
output += bias
if normalizer_fn:
output = normalizer_fn(output)
if gated:
output, gate = tf.split(output, 2, axis=-1)
if activation_fn:
output = activation_fn(output)
if gated:
output = output * tf.sigmoid(gate + gate_sigmoid_bias)
output = unflatten_from_ndims(output, s1, s2)
output = session.run(output)
return output
def check(self, x, padding, kernel, bias, strides):
"""Integrated tests for specific argument combinations."""
assert_allclose = functools.partial(np.testing.assert_allclose,
rtol=1e-5,
atol=1e-5)
strides = (strides, ) * 2 if is_integer(strides) else tuple(strides)
x_shape = (x.shape[-3], x.shape[-2])
x_channels = x.shape[-1]
kernel_size = kernel.shape[0], kernel.shape[1]
# compute the input for the deconv
y = Conv2dTestCase.conv2d_ans(x, padding, kernel, None, strides, 1)
y_shape = (y.shape[-3], y.shape[-2])
y_channels = y.shape[-1]
# test explicit output_shape, NHWC
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
x_shape,
padding,
kernel,
None,
strides,
channels_last=True,
use_bias=False)
self.assertEqual(deconv_out.shape, x.shape)
# memorize the linear output for later tests
linear_out = np.copy(deconv_out)
# test explicit output_shape, NCHW
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
x_shape,
padding,
kernel,
None,
strides,
channels_last=False,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test explicit dynamic output_shape, NHWC
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
tf.constant(x_shape),
padding,
kernel,
None,
strides,
channels_last=True,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test explicit dynamic output_shape, NCHW
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
tf.constant(x_shape),
padding,
kernel,
None,
strides,
channels_last=False,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test dynamic input, explicit dynamic output_shape, NHWC
ph = tf.placeholder(dtype=tf.float32,
shape=(None, ) * (len(y.shape) - 3) +
(None, None, y_channels))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
tf.constant(x_shape),
padding,
kernel,
None,
strides,
channels_last=True,
ph=ph,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test dynamic input, explicit dynamic output_shape, NCHW
ph = tf.placeholder(dtype=tf.float32,
shape=(None, ) * (len(y.shape) - 3) +
(y_channels, None, None))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
tf.constant(x_shape),
padding,
kernel,
None,
strides,
channels_last=False,
ph=ph,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# if the given payload shape matches the auto-inferred shape
# further test not giving explicit output_shape
def axis_matches(i):
return x_shape[i] == get_deconv_output_length(
y_shape[i], kernel_size[i], strides[i], padding)
if all(axis_matches(i) for i in (0, 1)):
# test static input, implicit output_shape, NHWC
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
None,
padding,
kernel,
None,
strides,
channels_last=True,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test static input, implicit output_shape, NCHW
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
None,
padding,
kernel,
None,
strides,
channels_last=False,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test dynamic input, implicit output_shape, NHWC
ph = tf.placeholder(dtype=tf.float32,
shape=(None, ) * (len(y.shape) - 3) +
(None, None, y_channels))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
None,
padding,
kernel,
None,
strides,
channels_last=True,
ph=ph,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test dynamic input, implicit output_shape, NCHW
ph = tf.placeholder(dtype=tf.float32,
shape=(None, ) * (len(y.shape) - 3) +
(y_channels, None, None))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
None,
padding,
kernel,
None,
strides,
channels_last=False,
ph=ph,
use_bias=False)
assert_allclose(deconv_out, linear_out)
# test normalization and activation
activation_fn = lambda x: x * 2. + 1.
normalizer_fn = lambda x: x * 1.5 - 3.
ans = activation_fn(normalizer_fn(linear_out))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
x_shape,
padding,
kernel,
bias,
strides,
channels_last=True,
normalizer_fn=normalizer_fn,
activation_fn=activation_fn)
assert_allclose(deconv_out, ans)
# test normalization and activation and force using bias
ans = activation_fn(normalizer_fn(linear_out + bias))
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
x_shape,
padding,
kernel,
bias,
strides,
channels_last=False,
use_bias=True,
normalizer_fn=normalizer_fn,
activation_fn=activation_fn)
assert_allclose(deconv_out, ans)
# test weight norm
normalized_kernel = l2_normalize(kernel, axis=(0, 1, 2))
ans = Deconv2dTestCase.run_deconv2d(y,
x_channels,
kernel_size,
x_shape,
padding,
normalized_kernel,
None,
strides,
channels_last=True,
use_bias=False)
deconv_out = Deconv2dTestCase.run_deconv2d(
y,
x_channels,
kernel_size,
x_shape,
padding,
kernel,
None,
strides,
channels_last=False,
use_bias=False,
weight_norm=True,
# this can force not using scale in weight_norm
normalizer_fn=(lambda x: x))
assert_allclose(deconv_out, ans)
# test gated
activation_fn = lambda x: x * 2. + 1.
normalizer_fn = lambda x: x * 1.5 - 3.
output, gate = np.split(normalizer_fn(linear_out), 2, axis=-1)
ans = activation_fn(output) * safe_sigmoid(gate + 1.1)
deconv_out = Deconv2dTestCase.run_deconv2d(y,
x_channels // 2,
kernel_size,
x_shape,
padding,
kernel,
bias,
strides,
channels_last=True,
normalizer_fn=normalizer_fn,
activation_fn=activation_fn,
gated=True,
gate_sigmoid_bias=1.1)
assert_allclose(deconv_out, ans)
def has_non_unit_item(x):
if is_integer(x):
return x != 1
else:
return any(i != 1 for i in x)
