def get_apply_compression(compression_op_spec, global_step):
"""Returns apply_compression operation matching compression_option input."""
compressor_spec = comp_op.LowRankDecompMatrixCompressor.get_default_hparams(
)
compressor_spec.set_hparam('rank', compression_op_spec.rank)
logging.info('Compressor spec %s', compressor_spec.to_json())
logging.info('Compression operator spec %s', compression_op_spec.to_json())
if compression_op_spec.compression_option not in _COMPRESSION_OPTIONS:
logging.info(
'Compression_option %s not in expected options: %s. '
'Will use low_rank decomp by default.',
str(compression_op_spec.compression_option),
','.join([str(opt) for opt in _COMPRESSION_OPTIONS]))
compression_op_spec.compression_option = 1
apply_compression = None
if compression_op_spec.compression_option == 1:
compressor = comp_op.LowRankDecompMatrixCompressor(
spec=compressor_spec)
apply_compression = comp_op.ApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
elif compression_op_spec.compression_option == 2:
compressor_spec.set_hparam('is_b_matrix_trainable', False)
compressor = simhash_comp_op.SimhashMatrixCompressor(
spec=compressor_spec)
apply_compression = simhash_comp_op.SimhashApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
elif compression_op_spec.compression_option == 3:
compressor_spec.set_hparam('is_b_matrix_trainable', False)
compressor_spec.set_hparam('use_lsh', True)
compressor = dl_compression_op.DLMatrixCompressor(spec=compressor_spec)
compression_op_spec.set_hparam('use_tpu', False)
apply_compression = dl_compression_op.DLApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
elif compression_op_spec.compression_option == 4:
compressor_spec.set_hparam('is_b_matrix_trainable', False)
compressor = simhash_comp_op.KmeansMatrixCompressor(
spec=compressor_spec)
apply_compression = simhash_comp_op.SimhashApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
return apply_compression
def test_DLDecompMatrixCompressor_interface(self):
compressor = dl_compression_op.DLMatrixCompressor(
spec=compression_op.LowRankDecompMatrixCompressor.get_default_hparams())
B = np.random.normal(0, 1, [20, 10]) # pylint: disable = invalid-name
sub_thresh_indices = (B <= 0.5)
B[sub_thresh_indices] = 0
B[:, 0] = np.ones(shape=B[:, 0].shape)
C = np.random.normal(0, 1, [10, 5]) # pylint: disable = invalid-name
A = np.matmul(B, C) # pylint: disable = invalid-name
[B_out, C_out] = compressor.static_matrix_compressor(A, n_iterations=32) # pylint: disable = invalid-name
A_recovered = np.matmul(B_out, C_out) # pylint: disable = invalid-name
print("np.linalg.norm(A-A_recovered) / np.linalg.norm(A): ",
np.linalg.norm(A - A_recovered) / np.linalg.norm(A))
print("A: ", A)
print("A_recovered: ", A_recovered)
print("fraction error np.linalg.norm(A-A_recovered): ",
np.linalg.norm(A - A_recovered) / np.linalg.norm(A))
self.assertLessEqual(
np.linalg.norm(A - A_recovered) / np.linalg.norm(A), 0.1)
def get_apply_compression(compression_op_spec, global_step):
"""Returns apply_compression operation matching compression_option input."""
compressor_spec = comp_op.LowRankDecompMatrixCompressor.get_default_hparams()
if compression_op_spec.__contains__('rank'):
compressor_spec.set_hparam('rank', compression_op_spec.rank)
if compression_op_spec.__contains__('block_size'):
compressor_spec.set_hparam('block_size', compression_op_spec.block_size)
logging.info('Compressor spec %s', compressor_spec.to_json())
logging.info('Compression operator spec %s', compression_op_spec.to_json())
if compression_op_spec.compression_option not in list(CompressionOptions):
# if unknown compression_option is given, default to low rank compression.
logging.info(
'Compression_option %s not in expected options: %s. '
'Will use low_rank decomp by default.',
str(compression_op_spec.compression_option),
','.join([str(opt) for opt in CompressionOptions]))
compression_op_spec.compression_option = CompressionOptions.LOWRANK_MATRIX_COMPRESSION
apply_compression = None
if compression_op_spec.compression_option == CompressionOptions.LOWRANK_MATRIX_COMPRESSION:
compressor = comp_op.LowRankDecompMatrixCompressor(spec=compressor_spec)
apply_compression = ApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
elif compression_op_spec.compression_option == CompressionOptions.SIMHASH_MATRIX_COMPRESSION:
compressor_spec.set_hparam('is_b_matrix_trainable', False)
compressor = simhash_comp_op.SimhashMatrixCompressor(spec=compressor_spec)
apply_compression = ApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
elif compression_op_spec.compression_option == CompressionOptions.KMEANS_MATRIX_COMPRESSION:
compressor_spec.set_hparam('is_b_matrix_trainable', True)
compressor = simhash_comp_op.KmeansMatrixCompressor(spec=compressor_spec)
apply_compression = ApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
elif compression_op_spec.compression_option == CompressionOptions.KMEANS_AND_PRUNING_MATRIX_COMPRESSION:
compressor_spec.set_hparam('is_b_matrix_trainable', True)
compressor = simhash_comp_op.KmeansMatrixCompressor(spec=compressor_spec)
apply_compression = ApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
elif compression_op_spec.compression_option == CompressionOptions.INPUTOUTPUT_COMPRESSION:
compressor_spec.set_hparam('is_b_matrix_trainable', True)
compressor_spec.set_hparam('is_c_matrix_trainable', True)
compressor_spec.set_hparam('is_d_matrix_trainable', True)
compressor = comp_op.LowRankDecompMatrixCompressor(spec=compressor_spec)
apply_compression = ApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
elif compression_op_spec.compression_option == CompressionOptions.BLOCK_COMPRESSION:
compressor_spec.set_hparam('is_c_matrix_trainable', True)
compressor = comp_op.LowRankDecompMatrixCompressor(spec=compressor_spec)
apply_compression = ApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
elif compression_op_spec.compression_option == CompressionOptions.MIXED_BLOCK_COMPRESSION:
compressor_spec.set_hparam('is_c_matrix_trainable', True)
compressor = comp_op.LowRankDecompMatrixCompressor(spec=compressor_spec)
apply_compression = ApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
elif compression_op_spec.compression_option == CompressionOptions.DL_MATRIX_COMPRESSION:
compressor = dl_comp_op.DLMatrixCompressor(spec=compressor_spec)
apply_compression = ApplyCompression(
scope='default_scope',
compression_spec=compression_op_spec,
compressor=compressor,
global_step=global_step)
return apply_compression
def check_dl_compression_op_interface(self, session, use_dl_op=False):
compression_hparams = ("name=cifar10_compression," +
"begin_compression_step=1000," +
"end_compression_step=120000," +
"compression_frequency=100," +
"compression_option=3," + "rank=200," +
"update_option=1")
update_style = 1
global_step = tf.get_variable("global_step", initializer=30)
compression_op_spec = compression_op.CompressionOp.get_default_hparams(
).parse(compression_hparams)
compression_op_spec.set_hparam("use_tpu", False)
if use_dl_op:
CompOp = dl_compression_op.DLCompressionOp # pylint: disable = invalid-name
else:
CompOp = compression_op.CompressionOp # pylint: disable = invalid-name
c = CompOp(spec=compression_op_spec, global_step=global_step)
# Need to add initial value for A so that we would know what to expect back.
code = np.random.normal(0, 1, [20, 10])
dictionary = np.random.normal(0, 1, [10, 5])
A_init = np.matmul(code, dictionary) # pylint: disable = invalid-name
A = tf.get_variable( # pylint: disable = invalid-name
"A",
initializer=A_init.astype(np.float32),
dtype=tf.float32)
MC = dl_compression_op.DLMatrixCompressor( # pylint: disable = invalid-name
spec=compression_op.LowRankDecompMatrixCompressor.
get_default_hparams().parse(
"num_rows=3,num_cols=3,rank=200,is_b_matrix_trainable=False"))
[_, A_update_op
] = c.get_apply_compression_op( # pylint: disable = invalid-name
A, MC, scope="my_scope")
tf.global_variables_initializer().run()
print("global_step: ", c._global_step.eval())
print("alpha: ", c.alpha.eval())
print("last_alpha_update_step: ", c._last_alpha_update_step.eval())
print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()),
np.linalg.norm(c.b_matrix_tfvar.eval()),
np.linalg.norm(c.c_matrix_tfvar.eval()))
self.assertAllEqual(
np.all(np.abs(np.linalg.norm(c.a_matrix_tfvar.eval())) < 0.00001),
False)
self.assertAllEqual(
np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001),
True)
tf.assign(global_step, 1001).eval()
print("global_step.eval is ", global_step.eval())
if update_style == 0:
A_update_op.eval()
else:
c.run_update_step(session)
print("global_step: ", c._global_step.eval())
print("alpha: ", c.alpha.eval())
print("last_alpha_update_step: ", c._last_alpha_update_step.eval())
print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()),
np.linalg.norm(c.b_matrix_tfvar.eval()),
np.linalg.norm(c.c_matrix_tfvar.eval()))
self.assertAllEqual(
np.all(np.abs(np.linalg.norm(c.b_matrix_tfvar.eval())) < 0.00001),
False)
self.assertAllEqual(
np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001),
False)
[B, C] = MC.static_matrix_compressor( # pylint: disable = invalid-name
c.a_matrix_tfvar.eval())
print("norm of error is ", np.linalg.norm(B - c.b_matrix_tfvar.eval()))
self.assertAllEqual(
np.all(np.abs(B - c.b_matrix_tfvar.eval()) < 0.00001), True)
self.assertAllEqual(
np.all(np.abs(C - c.c_matrix_tfvar.eval()) < 0.00001), True)
self.assertAllEqual(
np.all(np.abs(np.linalg.norm(c.b_matrix_tfvar.eval())) < 0.00001),
False)
self.assertAllEqual(
np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001),
False)
tf.assign(global_step, 1001).eval()
if update_style == 0:
A_update_op.eval()
else:
c.run_update_step(session)
print("global_step: ", c._global_step.eval())
print("alpha: ", c.alpha.eval())
print("last_alpha_update_step: ", c._last_alpha_update_step.eval())
print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()),
np.linalg.norm(c.b_matrix_tfvar.eval()),
np.linalg.norm(c.c_matrix_tfvar.eval()))
tf.assign(global_step, 2000).eval()
if update_style == 0:
A_update_op.eval()
else:
c.run_update_step(session)
print("global_step: ", c._global_step.eval())
print("alpha: ", c.alpha.eval())
print("last_alpha_update_step: ", c._last_alpha_update_step.eval())
self.assertAlmostEqual(c.alpha.eval(), 0.97)
self.assertEqual(c._last_alpha_update_step.eval(), 2000)
def check_dl_compression_op_interface_sparse(self,
session,
use_dl_op=False):
compression_hparams = ("name=cifar10_compression," +
"begin_compression_step=1000," +
"end_compression_step=120000," +
"compression_frequency=100," + "rank=200,")
update_style = compression_op_utils.UpdateOptions.PYTHON_UPDATE
global_step = tf.get_variable("global_step", initializer=30)
compression_op_spec = compression_op.CompressionOp.get_default_hparams(
).parse(compression_hparams)
compression_op_spec.set_hparam("use_tpu", False)
compression_op_spec.set_hparam(
"compression_option",
compression_op_utils.CompressionOptions.DL_MATRIX_COMPRESSION)
compression_op_spec.set_hparam(
"update_option", compression_op_utils.UpdateOptions.PYTHON_UPDATE)
if use_dl_op:
CompOp = dl_compression_op.DLCompressionOp # pylint: disable = invalid-name
else:
CompOp = compression_op.CompressionOp # pylint: disable = invalid-name
c = CompOp(spec=compression_op_spec, global_step=global_step)
code = np.random.normal(0, 1, [20, 10])
dictionary = np.random.normal(0, 1, [10, 5])
A_init = np.matmul(code, dictionary) # pylint: disable = invalid-name
A = tf.get_variable( # pylint: disable = invalid-name
"A",
initializer=A_init.astype(np.float32),
dtype=tf.float32)
MC = dl_compression_op.DLMatrixCompressor( # pylint: disable = invalid-name
spec=compression_op.LowRankDecompMatrixCompressor.
get_default_hparams().parse(
"num_rows=3,num_cols=3,rank=200,is_b_matrix_trainable=False"))
[_, A_update_op
] = c.get_apply_compression_op( # pylint: disable = invalid-name
A, MC, scope="my_scope")
tf.global_variables_initializer().run()
print("global_step: ", c._global_step.eval())
print("alpha: ", c.alpha.eval())
print("last_alpha_update_step: ", c._last_alpha_update_step.eval())
print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()),
c.b_matrix_indices_tfvar.eval().size,
np.linalg.norm(c.c_matrix_tfvar.eval()))
self.assertAllEqual(
np.all(np.abs(np.linalg.norm(c.a_matrix_tfvar.eval())) < 0.00001),
False)
self.assertAllEqual(
np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001),
True)
tf.assign(global_step, 1001).eval()
print("global_step.eval is ", global_step.eval())
if update_style == compression_op_utils.UpdateOptions.TF_UPDATE:
A_update_op.eval()
else:
c.run_update_step(session)
print("global_step: ", c._global_step.eval())
print("alpha: ", c.alpha.eval())
print("last_alpha_update_step: ", c._last_alpha_update_step.eval())
print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()),
c.b_matrix_indices_tfvar.eval().size,
np.linalg.norm(c.c_matrix_tfvar.eval()))
self.assertAllEqual(
np.all(c.b_matrix_indices_tfvar.eval().size < 0.00001), False)
self.assertAllEqual(
np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001),
False)
[B, C] = MC.static_matrix_compressor(
c.a_matrix_tfvar.eval()) # pylint: disable = invalid-name
print("B, B_tfvar :", B, c.b_matrix_tfvar.eval())
print("norm of error is ",
np.linalg.norm(B - tf.sparse.to_dense(c.b_matrix_tfvar).eval()))
self.assertAllEqual(
np.all(
np.abs(B -
tf.sparse.to_dense(c.b_matrix_tfvar).eval()) < 0.00001),
True)
self.assertAllEqual(
np.all(np.abs(C - c.c_matrix_tfvar.eval()) < 0.00001), True)
self.assertAllEqual(
np.all(c.b_matrix_indices_tfvar.eval().size < 0.00001), False)
self.assertAllEqual(
np.all(np.abs(np.linalg.norm(c.c_matrix_tfvar.eval())) < 0.00001),
False)
tf.assign(global_step, 1001).eval()
if update_style == compression_op_utils.UpdateOptions.TF_UPDATE:
A_update_op.eval()
else:
c.run_update_step(session)
print("global_step: ", c._global_step.eval())
print("alpha: ", c.alpha.eval())
print("last_alpha_update_step: ", c._last_alpha_update_step.eval())
print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()),
c.b_matrix_indices_tfvar.eval().size,
np.linalg.norm(c.c_matrix_tfvar.eval()))
tf.assign(global_step, 2000).eval()
if update_style == compression_op_utils.UpdateOptions.TF_UPDATE:
A_update_op.eval()
else:
c.run_update_step(session)
print("global_step: ", c._global_step.eval())
print("alpha: ", c.alpha.eval())
print("last_alpha_update_step: ", c._last_alpha_update_step.eval())
print("A,B,C norms are : ", np.linalg.norm(c.a_matrix_tfvar.eval()),
c.b_matrix_indices_tfvar.eval().size,
np.linalg.norm(c.c_matrix_tfvar.eval()))
self.assertAlmostEqual(c.alpha.eval(), 0.97)
self.assertEqual(c._last_alpha_update_step.eval(), 2000)
