def testMultiplyInverseAgainstExplicit(self):
with tf.Graph().as_default(), self.test_session() as sess:
tf.set_random_seed(200)
params = (tf.constant([1., 2.]), tf.constant(3.))
block = fb.NaiveDiagonalFB(lc.LayerCollection(), params)
block.register_additional_tower(32)
grads = (params[0]**2, tf.sqrt(params[1]))
damping = 0.5
block.instantiate_factors((grads, ), damping)
block._factor.instantiate_cov_variables()
cov = tf.reshape(tf.constant([2., 3., 4.]), [-1, 1])
sess.run(tf.assign(block._factor._cov, cov))
sess.run(block._factor.make_inverse_update_ops())
v_flat = np.array([4., 5., 6.], dtype=np.float32)
vector = utils.column_to_tensors(params, tf.constant(v_flat))
output = block.multiply_inverse(vector)
output_flat = sess.run(utils.tensors_to_column(output)).ravel()
full = sess.run(block.full_fisher_block())
explicit = np.dot(np.linalg.inv(full + damping * np.eye(3)),
v_flat)
self.assertAllClose(output_flat, explicit)
def testNaiveDiagonalFBInitTensorTuple(self):
with tf.Graph().as_default():
tf.set_random_seed(200)
params = (tf.constant([1., 2.]), tf.constant(3.))
block = fb.NaiveDiagonalFB(lc.LayerCollection(), params)
block.register_additional_tower(32)
self.assertAllEqual(params, block.tensors_to_compute_grads())
def testInstantiateFactors(self):
with tf.Graph().as_default():
tf.set_random_seed(200)
params = (tf.constant([1., 2.]), tf.constant(3.))
block = fb.NaiveDiagonalFB(lc.LayerCollection(), params)
block.register_additional_tower(32)
grads = (params[0]**2, tf.sqrt(params[1]))
block.instantiate_factors(grads, 0.5)
def testMultiplyInverseNotTuple(self):
with tf.Graph().as_default(), self.test_session() as sess:
tf.set_random_seed(200)
params = tf.constant([[1.], [2.]])
block = fb.NaiveDiagonalFB(lc.LayerCollection(), params)
block.register_additional_tower(32)
grads = params**2
block.instantiate_factors((grads,), 0.5)
block._factor.instantiate_cov_variables()
# Make sure our inverse is something other than the identity.
sess.run(tf.global_variables_initializer())
sess.run(block._factor.make_inverse_update_ops())
vector = tf.ones(2,) * 2
output = block.multiply_inverse(vector)
self.assertAllClose(sess.run(vector * 2 / 3.), sess.run(output))
