class DC_QSGDClient(Codec):
codex = quant(
build_quantization_space(
int(GlobalSettings.get_params(Quantization_Resolution_Client))))
def __init__(self, node_id):
super().__init__(node_id)
self.__gw_t: np.ndarray = 0
self.__lambda = 0.3
def dispose(self):
pass
def update_blocks(self,
block_weight: BlockWeight) -> netEncapsulation[IPack]:
self.__gw_t = block_weight.content
return netEncapsulation(Parameter_Server, SignalPack(self.node_id))
def receive_blocks(self, package: IPack) -> netEncapsulation[IPack]:
delta_w = package.decode(DC_QSGDServer.codex)
ggw_t = np.multiply(np.multiply(self.__gw_t, self.__gw_t), delta_w)
gw_t_tau = self.__gw_t + self.__lambda * ggw_t
self.set_result(gw_t_tau - delta_w, lambda x, y: x + y
if x is not None else y)
return netEncapsulation(
Parameter_Server,
QuantizedPack(self.node_id, gw_t_tau, DC_QSGDClient.codex))
def update_blocks(
self, block_weight: BlockWeight) -> netEncapsulation[NormalPack]:
package = NormalPack(
self.node_id,
block_weight.content.astype(
GlobalSettings.get_params(Full_Precision_Client)))
return netEncapsulation(Parameter_Server, package)
class QuantizedParaServer(Codec):
q_space = build_quantization_space(
int(GlobalSettings.get_params(Quantization_Resolution_Server)))
def __init__(self, node_id):
super().__init__(node_id)
self.__global_weights: np.ndarray = 0
self.__vt: np.ndarray = 0
self.__beta: float = 0.9
def dispose(self):
pass
def update_blocks(self, block_weight: BlockWeight):
pass
def receive_blocks(self, package: IPack):
grad = package.content
self.__global_weights -= grad
self.__vt = self.__beta * self.__vt + np.square(grad) * (1 -
self.__beta)
reply = QuantizedPack(
Parameter_Server,
*Q_w(self.__global_weights, QuantizedParaServer.q_space,
np.sqrt(self.__vt)))
return netEncapsulation(package.node_id, reply)
class DC_QSGDServer(Codec):
codex = quant(
build_quantization_space(
int(GlobalSettings.get_params(Quantization_Resolution_Server))))
def __init__(self, node_id):
super().__init__(node_id)
self.__global_weights: np.ndarray = 0
self.__weights_states: Dict[int, np.ndarray] = {}
def dispose(self):
pass
def update_blocks(self, block_weight: BlockWeight):
pass
def receive_blocks(self, package: IPack):
if isinstance(package, QuantizedPack):
self.__global_weights -= package.decode(DC_QSGDClient.codex)
self.__weights_states[package.node_id] = self.__global_weights
elif isinstance(package, SignalPack):
# returns Q(w_t+\tau - w_t)
if not isinstance(self.__global_weights, np.ndarray):
reply = SignalPack(Parameter_Server)
else:
reply = QuantizedPack(
Parameter_Server, self.__global_weights -
self.__weights_states.get(package.node_id, 0),
DC_QSGDServer.codex)
return netEncapsulation(package.node_id, reply)
def receive_blocks(self, package: IPack):
self.__global_weights -= package.content
reply = NormalPack(
Parameter_Server,
self.__global_weights.astype(
GlobalSettings.get_params(Low_Precision_Server)))
return netEncapsulation(package.node_id, reply)
def __init__(self, node_id):
super().__init__(node_id)
self.Global_State = 0
self.Weights_Last_Received = {}
for key in GlobalSettings.get_default().nodes:
self.Weights_Last_Received[key] = 0
def receive_blocks(
self, content: Tuple[int, ndarray]
) -> Union[Iterable[netEncapsulation], netEncapsulation, None]:
"""
PA Server receive a json_dict and send back a request
:param content:
:return:
"""
# update global current state
self.Bak_Weights_Node[content[0]] = content[1]
if len(self.Bak_Weights_Node) == GlobalSettings.get_default(
).node_count:
global_weight = np.mean(list(self.Bak_Weights_Node.values()),
axis=0)
self.dispose()
return netEncapsulation(GlobalSettings.get_default().nodes,
(Parameter_Server, global_weight))
def __do_grad_average(self):
how_much_nodes = GlobalSettings.get_default().node_count
if self.__current_recv == how_much_nodes:
# 执行梯度平均
self.set_result(self.__global_weights / how_much_nodes)
# 重设梯度值，等待下一批次的循环
self.__global_weights = np.asarray(0.0)
self.__current_recv = 0
def check_for_combine(self):
if len(self.BlockWeights) < GlobalSettings.get_default().block_count:
return
res = 0
for val in self.BlockWeights.values():
res += val
self.set_result(res / len(self.BlockWeights))
self.BlockWeights.clear()
def update_blocks(
self, block_weight: BlockWeight
) -> netEncapsulation[Tuple[int, ndarray]]:
"""
Try collect all blocks.
"""
self.BlockWeights[block_weight.block_id] = block_weight.content
self.check_for_combine()
send_to = GlobalSettings.get_default().get_adversary(
block_weight.block_id)
return netEncapsulation(send_to,
(block_weight.block_id, block_weight.content))
def build():
SGQPackage.__quant_codec = codec()
SGQPackage.__quant_space = build_quantization_space(
2**int(GlobalSettings.get_params(Quantization_Resolution_Client)) -
1)
SGQPackage.__quant_code = []
i = -1
while len(SGQPackage.__quant_code) != len(SGQPackage.__quant_space):
i += 1
flag = True
for c in SGQPackage.__quant_space:
if abs(c - i) < 1e-3:
flag = False
break
if flag:
SGQPackage.__quant_code.append(i)
SGQPackage.__quant_codec.set_codec(SGQPackage.__quant_code)
class QuantizedClient(Codec):
q_space = build_quantization_space(
int(GlobalSettings.get_params(Quantization_Resolution_Client)))
def __init__(self, node_id):
super().__init__(node_id)
def dispose(self):
pass
def update_blocks(
self,
block_weight: BlockWeight) -> netEncapsulation[QuantizedPack]:
package = QuantizedPack(
self.node_id, *Q_g(block_weight.content, QuantizedClient.q_space))
return netEncapsulation(Parameter_Server, package)
def receive_blocks(self, package: IPack):
self.set_result(package.content, lambda x, y: y)
def update_blocks(
self, block_weight: BlockWeight
) -> netEncapsulation[Dict[str, np.ndarray]]:
print('Weights delta received.')
print('from block: {}'.format(block_weight.block_id))
print('It has a content with shape: {}'.format(
block_weight.content.shape))
# 获取没有该数据的节点
send_to = GlobalSettings.get_default().get_adversary(
block_weight.block_id)
# 我们使用 'data' 字符串来标记我们的梯度内容
pkg = {'data': block_weight.content}
# 记录本机梯度
self.__global_weights += block_weight.content
self.__current_recv += 1
# 检查是否接受完所有数据
self.__do_grad_average()
# 发送梯度
return netEncapsulation(send_to, pkg)
class QuantizedParaServer(Codec):
codex = quant(
build_quantization_space(
int(GlobalSettings.get_params(Quantization_Resolution_Server))))
def __init__(self, node_id):
super().__init__(node_id)
self.__global_weights: np.ndarray = 0
def dispose(self):
pass
def update_blocks(self, block_weight: BlockWeight):
pass
def receive_blocks(self, package: IPack):
self.__global_weights -= package.decode(QuantizedClient.codex)
reply = QuantizedPack(Parameter_Server, self.__global_weights,
QuantizedParaServer.codex)
return netEncapsulation(package.node_id, reply)
def record(self, message: str):
from parallel_sgd.codec import GlobalSettings
GlobalSettings.global_logger().log_message(
"Codec: {}, Report: {}.".format(self.__class__.__name__, message))
# const parameters
SLAVE_CNT = 4
REDUNDANCY = 1
TEST_ROUNDS = 10
WEIGHTS_SHAPE = np.random.randint(3, 1024, size=2)
LAYER = 0
BATCHSIZE = 64
SYNCWAITTIMEOUT = 1000 #ms
# setup global parameters
GlobalSettings.deprecated_default_settings = DuplicateAssignment(
SLAVE_CNT, REDUNDANCY)
# default setting
Default = GlobalSettings.get_default()
class TestCodec(unittest.TestCase):
def test_p2p(self):
# build codec
slave_codec = [SLAVE_CODEC(node_id=i) for i in range(SLAVE_CNT)]
for i in range(TEST_ROUNDS):
# starting consensus stage
node_id = 0
for slave in slave_codec:
# build each block
for block_id in Default.node_2_block[node_id]:
# get random
arr = np.random.random(size=WEIGHTS_SHAPE)
def start(self, com: communication.Communication) -> None:
state, report = self.__check()
self.__log.log_message("Ready:{} \n\t Check List:\n\t\t--> {}".format(state, "\n\t\t--> ".join(report)))
# get dataset
train_x, train_y, test_x, test_y = self.__data.load()
self.__log.log_message('Dataset is ready, type: ({})'.format(self.__data))
# build data feeder
block_ids = GlobalSettings.get_default().node_2_block[self.node_id]
feeder = PSGDBlockDataFeeder(train_x, train_y, batch_iter=self.__batch_iter, block_ids=block_ids)
# assemble optimizer
self.__optimizer.assemble(transfer=self.__trans, block_mgr=feeder)
# compile model
self.__model.compile(self.__optimizer)
# summary
summary = self.__model.summary()
self.__log.log_message(summary)
trace_head = '{}-N({})'.format(self.__misc.mission_title, self.node_id)
self.__log.log_message('Model set to ready.')
log_head = self.__log.Title
# start !
GlobalSettings.deprecated_global_logger = self.__log
self.__trans.start_transfer(com, group_offset=list(self.group)[0], printer=self.__log, node_id=self.node_id)
# record data
time_start = time.time()
data_send_start = com.bytes_sent
data_recv_start = com.bytes_read
evaluation_history = []
title = []
r = {}
# this message will start the progress reporting
com.report_progress(0)
# do until reach the target accuracy
for i in range(self.__misc.epoch):
# change title
self.__log.Title = log_head + "-Epo-{}".format(i + 1)
history = self.__model.fit(feeder, epoch=1, printer=self.__log)
# do tests
r = self.__model.evaluate(test_x, test_y)
title = r.keys()
row = r.values()
self.__log.log_message('Evaluate result: {}'.format(r))
evaluation_history.append(row)
if self.__misc.target_acc is not None:
# only one metric in model metrics list.
# evaluation[0] refers to loss
# evaluation[1] refers to accuracy.
if r[1] > self.__misc.target_acc:
break
# report progress
com.report_progress(int(100 * ((i + 1) / self.__misc.epoch)))
# record data
time_end = time.time()
data_sent_end = com.bytes_sent
data_recv_end = com.bytes_read
training_history = self.__model.fit_history()
# save training history data
training_name = "TR-" + trace_head + ".csv"
training_trace = pd.DataFrame(training_history.history, columns=training_history.title)
training_trace.to_csv(training_name, index=False)
# save evaluation history data
evaluation_name = "EV-" + trace_head + ".csv"
evaluation_trace = pd.DataFrame(evaluation_history, columns=title)
evaluation_trace.to_csv(evaluation_name, index=False)
# save model
model_name = "MODEL-" + trace_head + ".model"
self.__model.compile(nn.gradient_descent.SGDOptimizer(learn_rate=1e-5))
self.__model.save(model_name)
self.__trace_filename.append(training_name)
self.__trace_filename.append(evaluation_name)
self.__trace_filename.append(model_name)
self.__log.log_message('Execution complete, time: {}.'.format(time_end - time_start))
self.__log.log_message('Execution complete, Total bytes sent: {}.'.format(data_sent_end - data_send_start))
self.__log.log_message('Execution complete, Total bytes read: {}.'.format(data_recv_end - data_recv_start))
self.__log.log_message('Trace file has been saved to {}.'.format(trace_head))
self.__log.flush()
# set marker
self.__done = True
# dispose
self.__model.clear()
del train_x, train_y, test_x, test_y
del self.__model, self.__log
# return last evaluation result
return r