def testMultiplyInverseAgainstExplicit(self): with tf.Graph().as_default(), self.test_session() as sess: tf.set_random_seed(200) params = (tf.constant([1., 2.]), tf.constant(3.)) block = fb.NaiveFullFB(lc.LayerCollection(), params) block.register_additional_tower(32) grads = (tf.constant([2., 3.]), tf.constant(4.)) damping = 0.5 block.instantiate_factors((grads,), damping) block._factor.instantiate_cov_variables() block.register_inverse() block._factor.instantiate_inv_variables() sess.run(tf.global_variables_initializer()) # Make sure our inverse is something other than the identity. sess.run(block._factor._cov.add_to_average(_make_psd(3))) sess.run(block._factor.make_inverse_update_ops()) v_flat = np.array([4., 5., 6.], dtype=np.float32) vector = utils.column_to_tensors(params, tf.constant(v_flat)) output = block.multiply_inverse(vector) output_flat = sess.run(utils.tensors_to_column(output)).ravel() full = sess.run(block.full_fisher_block()) explicit = np.dot(np.linalg.inv(full + damping * np.eye(3)), v_flat) self.assertAllClose(output_flat, explicit)
def testNaiveFullFBInitTensorTuple(self): with tf.Graph().as_default(): tf.set_random_seed(200) params = (tf.constant([1., 2.]), tf.constant(3.)) block = fb.NaiveFullFB(lc.LayerCollection(), params) block.register_additional_tower(32) self.assertAllEqual(params, block.tensors_to_compute_grads())
def testInstantiateFactors(self): with tf.Graph().as_default(): tf.set_random_seed(200) params = (tf.constant([1., 2.]), tf.constant(3.)) block = fb.NaiveFullFB(lc.LayerCollection(), params) block.register_additional_tower(32) grads = (params[0]**2, tf.sqrt(params[1])) block.instantiate_factors(grads, 0.5)
def testMultiplyInverseTuple(self): with tf.Graph().as_default(), self.test_session() as sess: tf.set_random_seed(200) params = (tf.constant([1., 2.]), tf.constant(3.)) block = fb.NaiveFullFB(lc.LayerCollection(), params) block.register_additional_tower(32) grads = (params[0]**2, tf.sqrt(params[1])) block.instantiate_factors((grads,), 0.5) block._factor.instantiate_cov_variables() block.register_inverse() block._factor.instantiate_inv_variables() # Make sure our inverse is something other than the identity. sess.run(tf.global_variables_initializer()) sess.run(block._factor.make_inverse_update_ops()) vector = tf.ones(3,) * 2 output = block.multiply_inverse(vector) self.assertAllClose(sess.run(vector * 2 / 3.), sess.run(output))