for epoch in range(num_epoch):
print(f"\nStart of Training Epoch {epoch}")
for step, (x_batch_train, y_batch_train) in enumerate(train_dataset):
with tf.GradientTape() as tape:
y_pred = model(x_batch_train, training=True)
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False)
loss = loss_fn(y_batch_train, y_pred)
train_accuracy(y_batch_train, y_pred)
train_loss(loss)
#print("Step", step, loss)
gradient_list = tape.gradient(loss, model.trainable_weights)
q = Quantize()
# Set the bitwidth here
q.bitwidth = 8
q_gradient_list = []
for each_array in gradient_list:
q_w = q.quantize(each_array.numpy())
q_gradient_list.append(tf.convert_to_tensor(q_w))
# TEST
'''
for each in range(len(q_gradient_list)):
print(q_gradient_list[each])
print("+++++++++++++++++++++++++++++")
print(gradient_list[each])
sys.exit()
'''
# Send QUANTIZED gradients to server
for epoch in range(num_epoch):
print(f"\nStart of Training Epoch {epoch}")
for step, (x_batch_train, y_batch_train) in enumerate(train_dataset):
with tf.GradientTape() as tape:
y_pred = model(x_batch_train, training=True)
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=False)
loss = loss_fn(y_batch_train, y_pred)
train_accuracy(y_batch_train, y_pred)
train_loss(loss)
#print("Step", step, loss)
gradient_list = tape.gradient(loss, model.trainable_weights)
q = Quantize()
# Set the bitwidth here
q.bitwidth = 16
q_gradient_list = []
for each_array in gradient_list:
q_w = q.quantize(each_array.numpy())
q_gradient_list.append(tf.convert_to_tensor(q_w))
# TEST
'''
for each in range(len(q_gradient_list)):
print(q_gradient_list[each])
print("+++++++++++++++++++++++++++++")
print(gradient_list[each])
sys.exit()
'''
# Send QUANTIZED gradients to server
threshold = tf.fill(grad_elem_shapes[i], k_val)
mask = tf.math.abs(grad[i]) < threshold
elems_equal = tf.equal(mask, False)
as_int = tf.cast(elems_equal, tf.int32)
count = tf.reduce_sum(as_int)
#print("COUNT K:", count)
np_u = np.array(u[i])
top_k_grad[i] = np.where(mask, 0.0, np_u)
# Send gradients to server
q = Quantize()
# Set the bitwidth here
q.bitwidth = 32
q_gradient_list = []
for each_array in top_k_grad:
q_w = q.quantize(each_array)
q_gradient_list.append(q_w)
for i in range(len(np_u)):
# Feedback error correction
r[i] = u[i] - q_gradient_list[i]
comm.send(q_gradient_list, dest=0, tag=11)
## NOT WORKING: Receive and set weights from server
#weights = comm.recv(source=0, tag=11)
#model.set_weights(weights)