def test_update_randomized_model(self):
artemis_update = ArtemisUpdate(self.params, self.workers)
artemis_update.workers_sub_set = [(self.workers[i], self.cost_models[i]) for i in range(self.params.nb_devices)]
artemis_update.H = torch.FloatTensor([-1 for i in range(10)])
artemis_update.omega = [torch.FloatTensor([i for i in range(0, 100, 10)]),
torch.FloatTensor([i for i in range(0,20, 2)])]
# Without momentum, should have no impact.
artemis_update.v = torch.FloatTensor([1 for i in range(10)])
artemis_update.update_randomized_model()
print(artemis_update.omega)
# Valid with the method of averaging
self.assertTrue(torch.all(artemis_update.v.eq(torch.FloatTensor([6*i - 1 for i in range(10)]))))
def test_doubleMODELcompression_WITH_memory(self):
params = MCM().define(n_dimensions=DIM, nb_devices=1,
quantization_param=10)
params.up_learning_rate = 0.5
workers = [Worker(0, params)]
workers[0].set_data(x, y)
workers[0].cost_model.L = workers[0].cost_model.local_L
update = ArtemisUpdate(params, workers)
artificial_l = ones_tensor.clone().detach()
update.H = artificial_l.clone().detach()
new_w = update.compute(w, 2, 2)
# Check that gradients have been updated.
self.assertFalse(torch.equal(update.g, zero_tensor))
self.assertFalse(torch.equal(update.v, zero_tensor))
self.assertFalse(torch.equal(update.h, zero_tensor))
# Check that l has been updated.
self.assertFalse(torch.equal(update.H, artificial_l))
# Check that correct value has been compressed
self.assertTrue(torch.equal(update.value_to_compress, new_w - artificial_l))
def test_doubleGRADIENTcompression_WITH_additional_memory(self):
params = DoreVariant().define(n_dimensions=DIM, nb_devices=1, quantization_param=10)
params.up_learning_rate = 0.5
workers = [Worker(0, params)]
workers[0].set_data(x, y)
workers[0].cost_model.L = workers[0].cost_model.local_L
update = ArtemisUpdate(params, workers)
artificial_l = ones_tensor.clone().detach()
# We artificially set different memory to check that it has impact on update computation.
update.H = artificial_l.clone().detach()
update.compute(w, 2, 2)
# Check that gradients have been updated.
self.assertFalse(torch.equal(update.g, zero_tensor))
self.assertFalse(torch.equal(update.v, zero_tensor))
self.assertFalse(torch.equal(update.h, zero_tensor))
# Check that l has been updated.
self.assertFalse(torch.equal(update.H, artificial_l))
# Check that correct value has been compressed
self.assertTrue(torch.equal(update.value_to_compress, update.g - artificial_l))