def test_send_back_global_informations_and_update(self):
artemis_update = ArtemisUpdate(self.params, self.workers)
self.workers[0].idx_last_update = 1
self.workers[1].idx_last_update = 1
artemis_update.workers_sub_set = [
(self.workers[i], self.cost_models[i])
for i in range(self.params.nb_devices)
]
artemis_update.omega_k = [[
torch.FloatTensor([0, 50]),
torch.FloatTensor([0, 10])
], [torch.FloatTensor([2, 4]),
torch.FloatTensor([10, 20])]]
nb_try = 1
artemis_update.step = 1 / 10
artemis_update.send_back_global_informations_and_update(
self.cost_models)
while (nb_try < 5 and torch.all(
artemis_update.workers[0].local_update.model_param.eq(
artemis_update.workers[1].local_update.model_param))):
self.workers[0].idx_last_update = 1
self.workers[1].idx_last_update = 1
artemis_update.send_back_global_informations_and_update(
self.cost_models)
nb_try += 1
self.assertFalse(
torch.all(artemis_update.workers[0].local_update.model_param.eq(
artemis_update.workers[1].local_update.model_param)),
"The models on workers are expected to be different.")
self.assertTrue(
self.workers[0].idx_last_update == 2
and self.workers[1].idx_last_update == 2,
"Index of last participation of each worker should be updated to 2"
)
def test_build_randomized_omega(self):
artemis_update = ArtemisUpdate(self.params, self.workers)
artemis_update.workers_sub_set = self.workers
artemis_update.value_to_compress = torch.FloatTensor(
[i for i in range(0, 100, 10)])
# We initilize omega_k with two values (as if we are at iteration 2)
artemis_update.omega_k = [[
torch.FloatTensor([i for i in range(0, 100, 10)]),
torch.FloatTensor([i for i in range(10)])
],
[
torch.FloatTensor(
[i for i in range(0, 20, 2)]),
torch.FloatTensor([i for i in range(10)])
]]
nb_try = 1
# We want to check that we have two different quantization of the value to compress.
# But in quantization there is some randomness, and thus vectors can some time be identical.
# We carry out five try, it after that there are still equal we consider that it is uncorrect.
artemis_update.build_randomized_omega(self.cost_models)
self.assertEqual(
len(artemis_update.omega), 2,
"The number of compressed value kept on central server must be equal to 2."
)
while (nb_try < 5 and torch.all(artemis_update.omega[0].eq(
artemis_update.omega[1]))):
artemis_update.build_randomized_omega(self.cost_models)
nb_try += 1
self.assertTrue(
nb_try < 5,
"After 5 try, the two different quantizations are still identical."
)
self.assertTrue(len(artemis_update.omega_k) == 3)
def test_initialization(self):
params = Parameters()
workers = [Worker(0, params)]
zero_tensor = torch.zeros(params.n_dimensions, dtype=np.float)
update = ArtemisUpdate(params, workers)
self.assertTrue(torch.equal(update.g, zero_tensor))
self.assertTrue(torch.equal(update.h, zero_tensor))
self.assertTrue(torch.equal(update.v, zero_tensor))
self.assertTrue(torch.equal(update.l, zero_tensor))
self.assertTrue(torch.equal(update.value_to_compress, zero_tensor))
def test_update_randomized_model(self):
artemis_update = ArtemisUpdate(self.params, self.workers)
artemis_update.workers_sub_set = [(self.workers[i], self.cost_models[i]) for i in range(self.params.nb_devices)]
artemis_update.H = torch.FloatTensor([-1 for i in range(10)])
artemis_update.omega = [torch.FloatTensor([i for i in range(0, 100, 10)]),
torch.FloatTensor([i for i in range(0,20, 2)])]
# Without momentum, should have no impact.
artemis_update.v = torch.FloatTensor([1 for i in range(10)])
artemis_update.update_randomized_model()
print(artemis_update.omega)
# Valid with the method of averaging
self.assertTrue(torch.all(artemis_update.v.eq(torch.FloatTensor([6*i - 1 for i in range(10)]))))
def test_Artemis(self):
params = Artemis().define(n_dimensions=DIM, nb_devices=1, quantization_param=10)
params.up_learning_rate = 0.5
workers = [Worker(0, params)]
workers[0].set_data(x, y)
workers[0].cost_model.L = workers[0].cost_model.local_L
update = ArtemisUpdate(params, workers)
update.compute(w, 2, 2)
# Check that gradients have been updated.
self.assertFalse(torch.equal(update.g, zero_tensor))
self.assertFalse(torch.equal(update.v, zero_tensor))
self.assertFalse(torch.equal(update.h, zero_tensor))
# Check that l has been updated.
self.assertTrue(torch.equal(update.H, zero_tensor))
# Check that correct value has been compressed
self.assertTrue(torch.equal(update.value_to_compress, update.g))
def test_Diana(self):
params = Diana().define(n_dimensions=DIM, nb_devices=1, quantization_param=10)
params.up_learning_rate = 0.5
workers = [Worker(0, params)]
workers[0].set_data(x, y)
workers[0].cost_model.L = workers[0].cost_model.local_L
update = ArtemisUpdate(params, workers)
new_model_param = update.compute(w, 2, 2)
# Check that gradients have been updated.
self.assertFalse(torch.equal(update.g, zero_tensor))
self.assertFalse(torch.equal(update.v, zero_tensor))
self.assertFalse(torch.equal(update.h, zero_tensor))
self.assertTrue(torch.equal(update.H, zero_tensor))
# Checking that for the return nothing has been quantized.
# there is a pb, with this test. Pass if ran with Artmis settings.
self.assertTrue(torch.equal(update.value_to_compress, zero_tensor))
def test_QSGD(self):
params = Qsgd().define(n_dimensions=DIM, nb_devices=1, quantization_param=10)
workers = [Worker(0, params)]
workers[0].set_data(x, y)
workers[0].cost_model.L = workers[0].cost_model.local_L
update = ArtemisUpdate(params, workers)
update.compute(w, 2, 2)
# Check that gradients have been updated.
# Check that gradients have been updated.
self.assertFalse(torch.equal(update.g, zero_tensor))
self.assertFalse(torch.equal(update.v, zero_tensor))
# Checking that no memory have been updated.
self.assertTrue(torch.equal(update.h, zero_tensor))
self.assertTrue(torch.equal(update.H, zero_tensor))
# Checking that for the return nothing has been quantized.
self.assertTrue(torch.equal(update.value_to_compress, zero_tensor))
def test_doubleMODELcompression_without_memory(self):
params = SGDDoubleModelCompressionWithoutMem().define(
n_dimensions=DIM, nb_devices=1, quantization_param=10)
params.learning_rate = 0.5
workers = [Worker(0, params)]
workers[0].set_data(x, y)
workers[0].cost_model.L = workers[0].cost_model.local_L
update = ArtemisUpdate(params, workers)
new_w = update.compute(w, 2, 2)
# Check that gradients have been updated.
self.assertFalse(torch.equal(update.g, zero_tensor))
self.assertFalse(torch.equal(update.v, zero_tensor))
self.assertFalse(torch.equal(update.h, zero_tensor))
# Check that l has been updated.
self.assertTrue(torch.equal(update.l, zero_tensor))
# Check that correct value has been compressed
self.assertTrue(torch.equal(update.value_to_compress, new_w))
def test_doubleMODELcompression_WITH_memory(self):
params = MCM().define(n_dimensions=DIM, nb_devices=1,
quantization_param=10)
params.up_learning_rate = 0.5
workers = [Worker(0, params)]
workers[0].set_data(x, y)
workers[0].cost_model.L = workers[0].cost_model.local_L
update = ArtemisUpdate(params, workers)
artificial_l = ones_tensor.clone().detach()
update.H = artificial_l.clone().detach()
new_w = update.compute(w, 2, 2)
# Check that gradients have been updated.
self.assertFalse(torch.equal(update.g, zero_tensor))
self.assertFalse(torch.equal(update.v, zero_tensor))
self.assertFalse(torch.equal(update.h, zero_tensor))
# Check that l has been updated.
self.assertFalse(torch.equal(update.H, artificial_l))
# Check that correct value has been compressed
self.assertTrue(torch.equal(update.value_to_compress, new_w - artificial_l))
def test_doubleGRADIENTcompression_WITH_additional_memory(self):
params = DoreVariant().define(n_dimensions=DIM, nb_devices=1, quantization_param=10)
params.up_learning_rate = 0.5
workers = [Worker(0, params)]
workers[0].set_data(x, y)
workers[0].cost_model.L = workers[0].cost_model.local_L
update = ArtemisUpdate(params, workers)
artificial_l = ones_tensor.clone().detach()
# We artificially set different memory to check that it has impact on update computation.
update.H = artificial_l.clone().detach()
update.compute(w, 2, 2)
# Check that gradients have been updated.
self.assertFalse(torch.equal(update.g, zero_tensor))
self.assertFalse(torch.equal(update.v, zero_tensor))
self.assertFalse(torch.equal(update.h, zero_tensor))
# Check that l has been updated.
self.assertFalse(torch.equal(update.H, artificial_l))
# Check that correct value has been compressed
self.assertTrue(torch.equal(update.value_to_compress, update.g - artificial_l))
def __update_method__(self) -> BasicGradientUpdate:
return ArtemisUpdate(self.parameters, self.workers)