def test_Artemis(self): params = Artemis().define(n_dimensions=DIM, nb_devices=1, quantization_param=10) params.up_learning_rate = 0.5 workers = [Worker(0, params)] workers[0].set_data(x, y) workers[0].cost_model.L = workers[0].cost_model.local_L update = ArtemisUpdate(params, workers) update.compute(w, 2, 2) # Check that gradients have been updated. self.assertFalse(torch.equal(update.g, zero_tensor)) self.assertFalse(torch.equal(update.v, zero_tensor)) self.assertFalse(torch.equal(update.h, zero_tensor)) # Check that l has been updated. self.assertTrue(torch.equal(update.H, zero_tensor)) # Check that correct value has been compressed self.assertTrue(torch.equal(update.value_to_compress, update.g))
def test_QSGD(self): params = Qsgd().define(n_dimensions=DIM, nb_devices=1, quantization_param=10) workers = [Worker(0, params)] workers[0].set_data(x, y) workers[0].cost_model.L = workers[0].cost_model.local_L update = ArtemisUpdate(params, workers) update.compute(w, 2, 2) # Check that gradients have been updated. # Check that gradients have been updated. self.assertFalse(torch.equal(update.g, zero_tensor)) self.assertFalse(torch.equal(update.v, zero_tensor)) # Checking that no memory have been updated. self.assertTrue(torch.equal(update.h, zero_tensor)) self.assertTrue(torch.equal(update.H, zero_tensor)) # Checking that for the return nothing has been quantized. self.assertTrue(torch.equal(update.value_to_compress, zero_tensor))
def test_doubleGRADIENTcompression_WITH_additional_memory(self): params = DoreVariant().define(n_dimensions=DIM, nb_devices=1, quantization_param=10) params.up_learning_rate = 0.5 workers = [Worker(0, params)] workers[0].set_data(x, y) workers[0].cost_model.L = workers[0].cost_model.local_L update = ArtemisUpdate(params, workers) artificial_l = ones_tensor.clone().detach() # We artificially set different memory to check that it has impact on update computation. update.H = artificial_l.clone().detach() update.compute(w, 2, 2) # Check that gradients have been updated. self.assertFalse(torch.equal(update.g, zero_tensor)) self.assertFalse(torch.equal(update.v, zero_tensor)) self.assertFalse(torch.equal(update.h, zero_tensor)) # Check that l has been updated. self.assertFalse(torch.equal(update.H, artificial_l)) # Check that correct value has been compressed self.assertTrue(torch.equal(update.value_to_compress, update.g - artificial_l))
def test_Diana(self): params = Diana().define(n_dimensions=DIM, nb_devices=1, quantization_param=10) params.up_learning_rate = 0.5 workers = [Worker(0, params)] workers[0].set_data(x, y) workers[0].cost_model.L = workers[0].cost_model.local_L update = ArtemisUpdate(params, workers) new_model_param = update.compute(w, 2, 2) # Check that gradients have been updated. self.assertFalse(torch.equal(update.g, zero_tensor)) self.assertFalse(torch.equal(update.v, zero_tensor)) self.assertFalse(torch.equal(update.h, zero_tensor)) self.assertTrue(torch.equal(update.H, zero_tensor)) # Checking that for the return nothing has been quantized. # there is a pb, with this test. Pass if ran with Artmis settings. self.assertTrue(torch.equal(update.value_to_compress, zero_tensor))
def test_doubleMODELcompression_without_memory(self): params = SGDDoubleModelCompressionWithoutMem().define( n_dimensions=DIM, nb_devices=1, quantization_param=10) params.learning_rate = 0.5 workers = [Worker(0, params)] workers[0].set_data(x, y) workers[0].cost_model.L = workers[0].cost_model.local_L update = ArtemisUpdate(params, workers) new_w = update.compute(w, 2, 2) # Check that gradients have been updated. self.assertFalse(torch.equal(update.g, zero_tensor)) self.assertFalse(torch.equal(update.v, zero_tensor)) self.assertFalse(torch.equal(update.h, zero_tensor)) # Check that l has been updated. self.assertTrue(torch.equal(update.l, zero_tensor)) # Check that correct value has been compressed self.assertTrue(torch.equal(update.value_to_compress, new_w))
def test_doubleMODELcompression_WITH_memory(self): params = MCM().define(n_dimensions=DIM, nb_devices=1, quantization_param=10) params.up_learning_rate = 0.5 workers = [Worker(0, params)] workers[0].set_data(x, y) workers[0].cost_model.L = workers[0].cost_model.local_L update = ArtemisUpdate(params, workers) artificial_l = ones_tensor.clone().detach() update.H = artificial_l.clone().detach() new_w = update.compute(w, 2, 2) # Check that gradients have been updated. self.assertFalse(torch.equal(update.g, zero_tensor)) self.assertFalse(torch.equal(update.v, zero_tensor)) self.assertFalse(torch.equal(update.h, zero_tensor)) # Check that l has been updated. self.assertFalse(torch.equal(update.H, artificial_l)) # Check that correct value has been compressed self.assertTrue(torch.equal(update.value_to_compress, new_w - artificial_l))