def order_4_hessian_penalty_test():
"""
Input:
x with shape (1, 2) (a two-dimensional vector with components x0 and x1)
Function:
G(x) = 0.25 * (x0 ** 2) * (x1 ** 2)
True Hessian Penalty:
(2 * x0 * x1) ** 2
"""
reset_tf()
x = tf.placeholder(shape=[1, 2], dtype=tf.float32)
with tf.name_scope('G_loss/GPU0'):
x_order_4 = tf.identity(0.25 * (x[:, 0]**2) * (x[:, 1]**2),
name='1x1/x_order_4')
hp_tf = multi_layer_hessian_penalty(G_z=x_order_4,
z=x,
num_rademacher_samples=16,
layers_to_reg=['1x1/x_order_4'],
current_lod=0.0,
max_lod=1.0,
gpu_ix=0,
log=False)
sess = tf.Session()
x_in = np.random.randn(1, 2)
gt = (2 * x_in[:, 0] * x_in[:, 1])[0]**2
hp_np = sess.run(hp_tf, feed_dict={x: x_in})
print(f'Hessian Penalty: {hp_np} | Ground Truth: {gt}')
def quadratic_hessian_penalty_test():
"""
Input:
x with shape (1, 2) (a two-dimensional vector with components x0 and x1)
Function:
G(x) = x0 * x1
True Hessian Penalty:
(1 + 1) ** 2 = 4.0
"""
reset_tf()
x = tf.placeholder(shape=[1, 2], dtype=tf.float32)
with tf.name_scope('G_loss/GPU0'):
x_quad = tf.identity(x[:, 0] * x[:, 1], name='1x1/x_quad')
hp_tf = multi_layer_hessian_penalty(G_z=x_quad,
z=x,
num_rademacher_samples=8,
layers_to_reg=['1x1/x_quad'],
current_lod=0.0,
max_lod=1.0,
gpu_ix=0,
log=False)
sess = tf.Session()
hp_np = sess.run(hp_tf, feed_dict={x: np.random.randn(1, 2)})
print(f'Hessian Penalty: {hp_np} | Ground Truth: 4.0')
def zero_hessian_penalty_test():
"""
Input:
scalar x
Function:
G(x) = x ** 2
True Hessian Penalty:
0.0 (since x is a scalar, there are no off-diagonal elements in G(x)'s Hessian)
"""
reset_tf()
x = tf.placeholder(shape=[], dtype=tf.float32)
with tf.name_scope('G_loss/GPU0'):
x_sqr = tf.identity(x**2, name='1x1/x_sqr')
hp_tf = multi_layer_hessian_penalty(G_z=x_sqr,
z=x,
num_rademacher_samples=4,
layers_to_reg=['1x1/x_sqr'],
current_lod=0.0,
max_lod=1.0,
gpu_ix=0,
log=False)
sess = tf.Session()
hp_np = sess.run(hp_tf, feed_dict={x: np.random.randn()})
print(f'Hessian Penalty: {hp_np} | Ground Truth: 0.0')
def batched_order_4_hessian_penalty_test():
"""
Input:
x with shape (4, 2) (batch of two-dimensional vectors)
Function:
G(x) = 0.25 * (x0 ** 2) * (x1 ** 2)
True Hessian Penalty (using max reduction):
max_{x0,x1} [(2 * x0 * x1) ** 2]
"""
reset_tf()
x = tf.placeholder(shape=[4, 2], dtype=tf.float32)
with tf.name_scope('G_loss/GPU0'):
x_order_4 = tf.identity(0.25 * (x[:, 0]**2) * (x[:, 1]**2),
name='1x1/x_order_4')
hp_tf = multi_layer_hessian_penalty(G_z=x_order_4,
z=x,
num_rademacher_samples=32,
layers_to_reg=['1x1/x_order_4'],
current_lod=0.0,
max_lod=1.0,
gpu_ix=0,
log=False)
sess = tf.Session()
x_in = np.random.randn(4, 2)
gt = np.max((2 * x_in[:, 0] * x_in[:, 1])**2, axis=0)
hp_np = sess.run(hp_tf, feed_dict={x: x_in})
print(f'Hessian Penalty: {hp_np} | Ground Truth: {gt}')
def G_wgan(G, D, opt, hp_lambda, HP_args, training_set, minibatch_size, lod_in,
max_lod, gpu_ix, **kwargs):
# Note: HP_args.hp_lambda is the MAXIMUM possible loss weighting of the Hessian Penalty (lambda in the paper).
# hp_lambda is the CURRENT loss weighting of the Hessian Penalty (lambda_t in the paper)
# Compute standard WGAN G loss:
latents = tf.random_normal([minibatch_size] + G.input_shapes[0][1:])
labels = training_set.get_random_labels_tf(minibatch_size)
fake_images_out = G.get_output_for(latents, labels, is_training=True)
fake_scores_out = fp32(
D.get_output_for(fake_images_out, labels, is_training=True))
loss = -fake_scores_out
if HP_args.hp_lambda > 0: # Compute Hessian Penalty for G:
hessian_penalty = multi_layer_hessian_penalty(
G_z=fake_images_out,
z=latents,
num_rademacher_samples=HP_args.num_rademacher_samples,
epsilon=HP_args.epsilon,
layers_to_reg=HP_args.layers_to_reg,
current_lod=lod_in,
max_lod=max_lod,
gpu_ix=gpu_ix)
# If we're fine-tuning with the Hessian Penalty, we don't want to start computing it until hp_start_nimg:
hessian_penalty = tf.cond(hp_lambda > 0, lambda: hessian_penalty,
lambda: 0.0)
hessian_penalty = autosummary('Loss/hessian_penalty', hessian_penalty)
else:
hessian_penalty = 0.0
G_loss = tf.reduce_mean(loss) + hp_lambda * hessian_penalty
return G_loss, hessian_penalty
def multi_layer_polynomial_hessian_penalty_test():
"""
Input:
x with shape (4, 2) (batch of two-dimensional vectors)
Function:
(Layer 1) G(x) = 0.25 * (x0 ** 2) * (x1 ** 2)
(Layer 2) F(x) = 10 * G(x)
True Hessian Penalty (using mean reduction):
(Layer 1) mean_{x0,x1} [(2 * x0 * x1) ** 2]
(Layer 2) mean_{x0,x1} [10 * (2 * x0 * x1)] ** 2
Returned Hessian Penalty = Layer 1 Hessian Penalty + Layer 2 Hessian Penalty
"""
reset_tf()
x = tf.placeholder(shape=[4, 2], dtype=tf.float32)
with tf.name_scope('G_loss/GPU0'):
x_order_4 = tf.identity(0.25 * (x[:, 0]**2) * (x[:, 1]**2),
name='1x1/x_order_4')
x_mul = tf.identity(10 * x_order_4, name='1x1/x_mul')
hp_tf = multi_layer_hessian_penalty(
G_z=x_mul,
z=x,
num_rademacher_samples=32,
layers_to_reg=['1x1/x_order_4', '1x1/x_mul'],
current_lod=0.0,
max_lod=1.0,
gpu_ix=0,
log=False,
reduction=tf.reduce_mean)
sess = tf.Session()
x_in = np.random.randn(4, 2)
first_layer_hp_before_squaring = (2 * x_in[:, 0] * x_in[:, 1])
gt = np.mean(first_layer_hp_before_squaring**2) + np.mean(
100 * first_layer_hp_before_squaring**2)
hp_np = sess.run(hp_tf, feed_dict={x: x_in})
print(f'Hessian Penalty: {hp_np} | Ground Truth: {gt}')