def testMultiplyInverseAgainstExplicit(self): with tf.Graph().as_default(), self.test_session() as sess: tf.set_random_seed(200) params = (tf.constant([1., 2.]), tf.constant(3.)) block = fb.NaiveDiagonalFB(lc.LayerCollection(), params) block.register_additional_tower(32) grads = (params[0]**2, tf.sqrt(params[1])) damping = 0.5 block.instantiate_factors((grads, ), damping) block._factor.instantiate_cov_variables() cov = tf.reshape(tf.constant([2., 3., 4.]), [-1, 1]) sess.run(tf.assign(block._factor._cov, cov)) sess.run(block._factor.make_inverse_update_ops()) v_flat = np.array([4., 5., 6.], dtype=np.float32) vector = utils.column_to_tensors(params, tf.constant(v_flat)) output = block.multiply_inverse(vector) output_flat = sess.run(utils.tensors_to_column(output)).ravel() full = sess.run(block.full_fisher_block()) explicit = np.dot(np.linalg.inv(full + damping * np.eye(3)), v_flat) self.assertAllClose(output_flat, explicit)
def testNaiveDiagonalFBInitTensorTuple(self): with tf.Graph().as_default(): tf.set_random_seed(200) params = (tf.constant([1., 2.]), tf.constant(3.)) block = fb.NaiveDiagonalFB(lc.LayerCollection(), params) block.register_additional_tower(32) self.assertAllEqual(params, block.tensors_to_compute_grads())
def testInstantiateFactors(self): with tf.Graph().as_default(): tf.set_random_seed(200) params = (tf.constant([1., 2.]), tf.constant(3.)) block = fb.NaiveDiagonalFB(lc.LayerCollection(), params) block.register_additional_tower(32) grads = (params[0]**2, tf.sqrt(params[1])) block.instantiate_factors(grads, 0.5)
def testMultiplyInverseNotTuple(self): with tf.Graph().as_default(), self.test_session() as sess: tf.set_random_seed(200) params = tf.constant([[1.], [2.]]) block = fb.NaiveDiagonalFB(lc.LayerCollection(), params) block.register_additional_tower(32) grads = params**2 block.instantiate_factors((grads,), 0.5) block._factor.instantiate_cov_variables() # Make sure our inverse is something other than the identity. sess.run(tf.global_variables_initializer()) sess.run(block._factor.make_inverse_update_ops()) vector = tf.ones(2,) * 2 output = block.multiply_inverse(vector) self.assertAllClose(sess.run(vector * 2 / 3.), sess.run(output))