def test_update_randomized_model(self): artemis_update = ArtemisUpdate(self.params, self.workers) artemis_update.workers_sub_set = [(self.workers[i], self.cost_models[i]) for i in range(self.params.nb_devices)] artemis_update.H = torch.FloatTensor([-1 for i in range(10)]) artemis_update.omega = [torch.FloatTensor([i for i in range(0, 100, 10)]), torch.FloatTensor([i for i in range(0,20, 2)])] # Without momentum, should have no impact. artemis_update.v = torch.FloatTensor([1 for i in range(10)]) artemis_update.update_randomized_model() print(artemis_update.omega) # Valid with the method of averaging self.assertTrue(torch.all(artemis_update.v.eq(torch.FloatTensor([6*i - 1 for i in range(10)]))))
def test_doubleMODELcompression_WITH_memory(self): params = MCM().define(n_dimensions=DIM, nb_devices=1, quantization_param=10) params.up_learning_rate = 0.5 workers = [Worker(0, params)] workers[0].set_data(x, y) workers[0].cost_model.L = workers[0].cost_model.local_L update = ArtemisUpdate(params, workers) artificial_l = ones_tensor.clone().detach() update.H = artificial_l.clone().detach() new_w = update.compute(w, 2, 2) # Check that gradients have been updated. self.assertFalse(torch.equal(update.g, zero_tensor)) self.assertFalse(torch.equal(update.v, zero_tensor)) self.assertFalse(torch.equal(update.h, zero_tensor)) # Check that l has been updated. self.assertFalse(torch.equal(update.H, artificial_l)) # Check that correct value has been compressed self.assertTrue(torch.equal(update.value_to_compress, new_w - artificial_l))
def test_doubleGRADIENTcompression_WITH_additional_memory(self): params = DoreVariant().define(n_dimensions=DIM, nb_devices=1, quantization_param=10) params.up_learning_rate = 0.5 workers = [Worker(0, params)] workers[0].set_data(x, y) workers[0].cost_model.L = workers[0].cost_model.local_L update = ArtemisUpdate(params, workers) artificial_l = ones_tensor.clone().detach() # We artificially set different memory to check that it has impact on update computation. update.H = artificial_l.clone().detach() update.compute(w, 2, 2) # Check that gradients have been updated. self.assertFalse(torch.equal(update.g, zero_tensor)) self.assertFalse(torch.equal(update.v, zero_tensor)) self.assertFalse(torch.equal(update.h, zero_tensor)) # Check that l has been updated. self.assertFalse(torch.equal(update.H, artificial_l)) # Check that correct value has been compressed self.assertTrue(torch.equal(update.value_to_compress, update.g - artificial_l))