def get_apply_compression(compression_op_spec, global_step): """Returns apply_compression operation matching compression_option input.""" compressor_spec = comp_op.LowRankDecompMatrixCompressor.get_default_hparams( ) compressor_spec.set_hparam('rank', compression_op_spec.rank) logging.info('Compressor spec %s', compressor_spec.to_json()) logging.info('Compression operator spec %s', compression_op_spec.to_json()) if compression_op_spec.compression_option not in _COMPRESSION_OPTIONS: logging.info( 'Compression_option %s not in expected options: %s. ' 'Will use low_rank decomp by default.', str(compression_op_spec.compression_option), ','.join([str(opt) for opt in _COMPRESSION_OPTIONS])) compression_op_spec.compression_option = 1 apply_compression = None if compression_op_spec.compression_option == 1: compressor = comp_op.LowRankDecompMatrixCompressor( spec=compressor_spec) apply_compression = comp_op.ApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) elif compression_op_spec.compression_option == 2: compressor_spec.set_hparam('is_b_matrix_trainable', False) compressor = simhash_comp_op.SimhashMatrixCompressor( spec=compressor_spec) apply_compression = simhash_comp_op.SimhashApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) elif compression_op_spec.compression_option == 3: compressor_spec.set_hparam('is_b_matrix_trainable', False) compressor_spec.set_hparam('use_lsh', True) compressor = dl_compression_op.DLMatrixCompressor(spec=compressor_spec) compression_op_spec.set_hparam('use_tpu', False) apply_compression = dl_compression_op.DLApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) elif compression_op_spec.compression_option == 4: compressor_spec.set_hparam('is_b_matrix_trainable', False) compressor = simhash_comp_op.KmeansMatrixCompressor( spec=compressor_spec) apply_compression = simhash_comp_op.SimhashApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) return apply_compression
def test_DLDecompMatrixCompressor_interface(self): compressor = dl_compression_op.DLMatrixCompressor( spec=compression_op.LowRankDecompMatrixCompressor.get_default_hparams()) B = np.random.normal(0, 1, [20, 10]) # pylint: disable = invalid-name sub_thresh_indices = (B <= 0.5) B[sub_thresh_indices] = 0 B[:, 0] = np.ones(shape=B[:, 0].shape) C = np.random.normal(0, 1, [10, 5]) # pylint: disable = invalid-name A = np.matmul(B, C) # pylint: disable = invalid-name [B_out, C_out] = compressor.static_matrix_compressor(A, n_iterations=32) # pylint: disable = invalid-name A_recovered = np.matmul(B_out, C_out) # pylint: disable = invalid-name print("np.linalg.norm(A-A_recovered) / np.linalg.norm(A): ", np.linalg.norm(A - A_recovered) / np.linalg.norm(A)) print("A: ", A) print("A_recovered: ", A_recovered) print("fraction error np.linalg.norm(A-A_recovered): ", np.linalg.norm(A - A_recovered) / np.linalg.norm(A)) self.assertLessEqual( np.linalg.norm(A - A_recovered) / np.linalg.norm(A), 0.1)
def get_apply_compression(compression_op_spec, global_step): """Returns apply_compression operation matching compression_option input.""" compressor_spec = comp_op.LowRankDecompMatrixCompressor.get_default_hparams() if compression_op_spec.__contains__('rank'): compressor_spec.set_hparam('rank', compression_op_spec.rank) if compression_op_spec.__contains__('block_size'): compressor_spec.set_hparam('block_size', compression_op_spec.block_size) logging.info('Compressor spec %s', compressor_spec.to_json()) logging.info('Compression operator spec %s', compression_op_spec.to_json()) if compression_op_spec.compression_option not in list(CompressionOptions): # if unknown compression_option is given, default to low rank compression. logging.info( 'Compression_option %s not in expected options: %s. ' 'Will use low_rank decomp by default.', str(compression_op_spec.compression_option), ','.join([str(opt) for opt in CompressionOptions])) compression_op_spec.compression_option = CompressionOptions.LOWRANK_MATRIX_COMPRESSION apply_compression = None if compression_op_spec.compression_option == CompressionOptions.LOWRANK_MATRIX_COMPRESSION: compressor = comp_op.LowRankDecompMatrixCompressor(spec=compressor_spec) apply_compression = ApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) elif compression_op_spec.compression_option == CompressionOptions.SIMHASH_MATRIX_COMPRESSION: compressor_spec.set_hparam('is_b_matrix_trainable', False) compressor = simhash_comp_op.SimhashMatrixCompressor(spec=compressor_spec) apply_compression = ApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) elif compression_op_spec.compression_option == CompressionOptions.KMEANS_MATRIX_COMPRESSION: compressor_spec.set_hparam('is_b_matrix_trainable', True) compressor = simhash_comp_op.KmeansMatrixCompressor(spec=compressor_spec) apply_compression = ApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) elif compression_op_spec.compression_option == CompressionOptions.KMEANS_AND_PRUNING_MATRIX_COMPRESSION: compressor_spec.set_hparam('is_b_matrix_trainable', True) compressor = simhash_comp_op.KmeansMatrixCompressor(spec=compressor_spec) apply_compression = ApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) elif compression_op_spec.compression_option == CompressionOptions.INPUTOUTPUT_COMPRESSION: compressor_spec.set_hparam('is_b_matrix_trainable', True) compressor_spec.set_hparam('is_c_matrix_trainable', True) compressor_spec.set_hparam('is_d_matrix_trainable', True) compressor = comp_op.LowRankDecompMatrixCompressor(spec=compressor_spec) apply_compression = ApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) elif compression_op_spec.compression_option == CompressionOptions.BLOCK_COMPRESSION: compressor_spec.set_hparam('is_c_matrix_trainable', True) compressor = comp_op.LowRankDecompMatrixCompressor(spec=compressor_spec) apply_compression = ApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) elif compression_op_spec.compression_option == CompressionOptions.MIXED_BLOCK_COMPRESSION: compressor_spec.set_hparam('is_c_matrix_trainable', True) compressor = comp_op.LowRankDecompMatrixCompressor(spec=compressor_spec) apply_compression = ApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) elif compression_op_spec.compression_option == CompressionOptions.DL_MATRIX_COMPRESSION: compressor = dl_comp_op.DLMatrixCompressor(spec=compressor_spec) apply_compression = ApplyCompression( scope='default_scope', compression_spec=compression_op_spec, compressor=compressor, global_step=global_step) return apply_compression
def check_dl_compression_op_interface(self, session, use_dl_op=False): compression_hparams = ("name=cifar10_compression," + "begin_compression_step=1000," + "end_compression_step=120000," + "compression_frequency=100," + "compression_option=3," + "rank=200," + "update_option=1") update_style = 1 global_step = tf.get_variable("global_step", initializer=30) compression_op_spec = compression_op.CompressionOp.get_default_hparams( ).parse(compression_hparams) compression_op_spec.set_hparam("use_tpu", False) if use_dl_op: CompOp = dl_compression_op.DLCompressionOp # pylint: disable = invalid-name else: CompOp = compression_op.CompressionOp # pylint: disable = invalid-name c = CompOp(spec=compression_op_spec, global_step=global_step) # Need to add initial value for A so that we would know what to expect back. code = np.random.normal(0, 1, [20, 10]) dictionary = np.random.normal(0, 1, [10, 5]) A_init = np.matmul(code, dictionary) # pylint: disable = invalid-name A = tf.get_variable( # pylint: disable = invalid-name "A", initializer=A_init.astype(np.float32), dtype=tf.float32) MC = dl_compression_op.DLMatrixCompressor( # pylint: disable = invalid-name spec=compression_op.LowRankDecompMatrixCompressor. get_default_hparams().parse( "num_rows=3,num_cols=3,rank=200,is_b_matrix_trainable=False")) [_, A_update_op ] = c.get_apply_compression_op( # pylint: disable = invalid-name A, MC, scope="my_scope") tf.global_variables_initializer().run() print("global_step: ", c._global_step.eval()) print("alpha: ", c.alpha.eval()) print("last_alpha_update_step: ", c._last_alpha_update_step.eval()) print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()), np.linalg.norm(c.b_matrix_tfvar.eval()), np.linalg.norm(c.c_matrix_tfvar.eval())) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.a_matrix_tfvar.eval())) < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), True) tf.assign(global_step, 1001).eval() print("global_step.eval is ", global_step.eval()) if update_style == 0: A_update_op.eval() else: c.run_update_step(session) print("global_step: ", c._global_step.eval()) print("alpha: ", c.alpha.eval()) print("last_alpha_update_step: ", c._last_alpha_update_step.eval()) print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()), np.linalg.norm(c.b_matrix_tfvar.eval()), np.linalg.norm(c.c_matrix_tfvar.eval())) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.b_matrix_tfvar.eval())) < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), False) [B, C] = MC.static_matrix_compressor( # pylint: disable = invalid-name c.a_matrix_tfvar.eval()) print("norm of error is ", np.linalg.norm(B - c.b_matrix_tfvar.eval())) self.assertAllEqual( np.all(np.abs(B - c.b_matrix_tfvar.eval()) < 0.00001), True) self.assertAllEqual( np.all(np.abs(C - c.c_matrix_tfvar.eval()) < 0.00001), True) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.b_matrix_tfvar.eval())) < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), False) tf.assign(global_step, 1001).eval() if update_style == 0: A_update_op.eval() else: c.run_update_step(session) print("global_step: ", c._global_step.eval()) print("alpha: ", c.alpha.eval()) print("last_alpha_update_step: ", c._last_alpha_update_step.eval()) print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()), np.linalg.norm(c.b_matrix_tfvar.eval()), np.linalg.norm(c.c_matrix_tfvar.eval())) tf.assign(global_step, 2000).eval() if update_style == 0: A_update_op.eval() else: c.run_update_step(session) print("global_step: ", c._global_step.eval()) print("alpha: ", c.alpha.eval()) print("last_alpha_update_step: ", c._last_alpha_update_step.eval()) self.assertAlmostEqual(c.alpha.eval(), 0.97) self.assertEqual(c._last_alpha_update_step.eval(), 2000)
def check_dl_compression_op_interface_sparse(self, session, use_dl_op=False): compression_hparams = ("name=cifar10_compression," + "begin_compression_step=1000," + "end_compression_step=120000," + "compression_frequency=100," + "rank=200,") update_style = compression_op_utils.UpdateOptions.PYTHON_UPDATE global_step = tf.get_variable("global_step", initializer=30) compression_op_spec = compression_op.CompressionOp.get_default_hparams( ).parse(compression_hparams) compression_op_spec.set_hparam("use_tpu", False) compression_op_spec.set_hparam( "compression_option", compression_op_utils.CompressionOptions.DL_MATRIX_COMPRESSION) compression_op_spec.set_hparam( "update_option", compression_op_utils.UpdateOptions.PYTHON_UPDATE) if use_dl_op: CompOp = dl_compression_op.DLCompressionOp # pylint: disable = invalid-name else: CompOp = compression_op.CompressionOp # pylint: disable = invalid-name c = CompOp(spec=compression_op_spec, global_step=global_step) code = np.random.normal(0, 1, [20, 10]) dictionary = np.random.normal(0, 1, [10, 5]) A_init = np.matmul(code, dictionary) # pylint: disable = invalid-name A = tf.get_variable( # pylint: disable = invalid-name "A", initializer=A_init.astype(np.float32), dtype=tf.float32) MC = dl_compression_op.DLMatrixCompressor( # pylint: disable = invalid-name spec=compression_op.LowRankDecompMatrixCompressor. get_default_hparams().parse( "num_rows=3,num_cols=3,rank=200,is_b_matrix_trainable=False")) [_, A_update_op ] = c.get_apply_compression_op( # pylint: disable = invalid-name A, MC, scope="my_scope") tf.global_variables_initializer().run() print("global_step: ", c._global_step.eval()) print("alpha: ", c.alpha.eval()) print("last_alpha_update_step: ", c._last_alpha_update_step.eval()) print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()), c.b_matrix_indices_tfvar.eval().size, np.linalg.norm(c.c_matrix_tfvar.eval())) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.a_matrix_tfvar.eval())) < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), True) tf.assign(global_step, 1001).eval() print("global_step.eval is ", global_step.eval()) if update_style == compression_op_utils.UpdateOptions.TF_UPDATE: A_update_op.eval() else: c.run_update_step(session) print("global_step: ", c._global_step.eval()) print("alpha: ", c.alpha.eval()) print("last_alpha_update_step: ", c._last_alpha_update_step.eval()) print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()), c.b_matrix_indices_tfvar.eval().size, np.linalg.norm(c.c_matrix_tfvar.eval())) self.assertAllEqual( np.all(c.b_matrix_indices_tfvar.eval().size < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), False) [B, C] = MC.static_matrix_compressor( c.a_matrix_tfvar.eval()) # pylint: disable = invalid-name print("B, B_tfvar :", B, c.b_matrix_tfvar.eval()) print("norm of error is ", np.linalg.norm(B - tf.sparse.to_dense(c.b_matrix_tfvar).eval())) self.assertAllEqual( np.all( np.abs(B - tf.sparse.to_dense(c.b_matrix_tfvar).eval()) < 0.00001), True) self.assertAllEqual( np.all(np.abs(C - c.c_matrix_tfvar.eval()) < 0.00001), True) self.assertAllEqual( np.all(c.b_matrix_indices_tfvar.eval().size < 0.00001), False) self.assertAllEqual( np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001), False) tf.assign(global_step, 1001).eval() if update_style == compression_op_utils.UpdateOptions.TF_UPDATE: A_update_op.eval() else: c.run_update_step(session) print("global_step: ", c._global_step.eval()) print("alpha: ", c.alpha.eval()) print("last_alpha_update_step: ", c._last_alpha_update_step.eval()) print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()), c.b_matrix_indices_tfvar.eval().size, np.linalg.norm(c.c_matrix_tfvar.eval())) tf.assign(global_step, 2000).eval() if update_style == compression_op_utils.UpdateOptions.TF_UPDATE: A_update_op.eval() else: c.run_update_step(session) print("global_step: ", c._global_step.eval()) print("alpha: ", c.alpha.eval()) print("last_alpha_update_step: ", c._last_alpha_update_step.eval()) print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()), c.b_matrix_indices_tfvar.eval().size, np.linalg.norm(c.c_matrix_tfvar.eval())) self.assertAlmostEqual(c.alpha.eval(), 0.97) self.assertEqual(c._last_alpha_update_step.eval(), 2000)