class DC_QSGDClient(Codec): codex = quant( build_quantization_space( int(GlobalSettings.get_params(Quantization_Resolution_Client)))) def __init__(self, node_id): super().__init__(node_id) self.__gw_t: np.ndarray = 0 self.__lambda = 0.3 def dispose(self): pass def update_blocks(self, block_weight: BlockWeight) -> netEncapsulation[IPack]: self.__gw_t = block_weight.content return netEncapsulation(Parameter_Server, SignalPack(self.node_id)) def receive_blocks(self, package: IPack) -> netEncapsulation[IPack]: delta_w = package.decode(DC_QSGDServer.codex) ggw_t = np.multiply(np.multiply(self.__gw_t, self.__gw_t), delta_w) gw_t_tau = self.__gw_t + self.__lambda * ggw_t self.set_result(gw_t_tau - delta_w, lambda x, y: x + y if x is not None else y) return netEncapsulation( Parameter_Server, QuantizedPack(self.node_id, gw_t_tau, DC_QSGDClient.codex))
def update_blocks( self, block_weight: BlockWeight) -> netEncapsulation[NormalPack]: package = NormalPack( self.node_id, block_weight.content.astype( GlobalSettings.get_params(Full_Precision_Client))) return netEncapsulation(Parameter_Server, package)
class QuantizedParaServer(Codec): q_space = build_quantization_space( int(GlobalSettings.get_params(Quantization_Resolution_Server))) def __init__(self, node_id): super().__init__(node_id) self.__global_weights: np.ndarray = 0 self.__vt: np.ndarray = 0 self.__beta: float = 0.9 def dispose(self): pass def update_blocks(self, block_weight: BlockWeight): pass def receive_blocks(self, package: IPack): grad = package.content self.__global_weights -= grad self.__vt = self.__beta * self.__vt + np.square(grad) * (1 - self.__beta) reply = QuantizedPack( Parameter_Server, *Q_w(self.__global_weights, QuantizedParaServer.q_space, np.sqrt(self.__vt))) return netEncapsulation(package.node_id, reply)
class DC_QSGDServer(Codec): codex = quant( build_quantization_space( int(GlobalSettings.get_params(Quantization_Resolution_Server)))) def __init__(self, node_id): super().__init__(node_id) self.__global_weights: np.ndarray = 0 self.__weights_states: Dict[int, np.ndarray] = {} def dispose(self): pass def update_blocks(self, block_weight: BlockWeight): pass def receive_blocks(self, package: IPack): if isinstance(package, QuantizedPack): self.__global_weights -= package.decode(DC_QSGDClient.codex) self.__weights_states[package.node_id] = self.__global_weights elif isinstance(package, SignalPack): # returns Q(w_t+\tau - w_t) if not isinstance(self.__global_weights, np.ndarray): reply = SignalPack(Parameter_Server) else: reply = QuantizedPack( Parameter_Server, self.__global_weights - self.__weights_states.get(package.node_id, 0), DC_QSGDServer.codex) return netEncapsulation(package.node_id, reply)
def receive_blocks(self, package: IPack): self.__global_weights -= package.content reply = NormalPack( Parameter_Server, self.__global_weights.astype( GlobalSettings.get_params(Low_Precision_Server))) return netEncapsulation(package.node_id, reply)
def __init__(self, node_id): super().__init__(node_id) self.Global_State = 0 self.Weights_Last_Received = {} for key in GlobalSettings.get_default().nodes: self.Weights_Last_Received[key] = 0
def receive_blocks( self, content: Tuple[int, ndarray] ) -> Union[Iterable[netEncapsulation], netEncapsulation, None]: """ PA Server receive a json_dict and send back a request :param content: :return: """ # update global current state self.Bak_Weights_Node[content[0]] = content[1] if len(self.Bak_Weights_Node) == GlobalSettings.get_default( ).node_count: global_weight = np.mean(list(self.Bak_Weights_Node.values()), axis=0) self.dispose() return netEncapsulation(GlobalSettings.get_default().nodes, (Parameter_Server, global_weight))
def __do_grad_average(self): how_much_nodes = GlobalSettings.get_default().node_count if self.__current_recv == how_much_nodes: # 执行梯度平均 self.set_result(self.__global_weights / how_much_nodes) # 重设梯度值,等待下一批次的循环 self.__global_weights = np.asarray(0.0) self.__current_recv = 0
def check_for_combine(self): if len(self.BlockWeights) < GlobalSettings.get_default().block_count: return res = 0 for val in self.BlockWeights.values(): res += val self.set_result(res / len(self.BlockWeights)) self.BlockWeights.clear()
def update_blocks( self, block_weight: BlockWeight ) -> netEncapsulation[Tuple[int, ndarray]]: """ Try collect all blocks. """ self.BlockWeights[block_weight.block_id] = block_weight.content self.check_for_combine() send_to = GlobalSettings.get_default().get_adversary( block_weight.block_id) return netEncapsulation(send_to, (block_weight.block_id, block_weight.content))
def build(): SGQPackage.__quant_codec = codec() SGQPackage.__quant_space = build_quantization_space( 2**int(GlobalSettings.get_params(Quantization_Resolution_Client)) - 1) SGQPackage.__quant_code = [] i = -1 while len(SGQPackage.__quant_code) != len(SGQPackage.__quant_space): i += 1 flag = True for c in SGQPackage.__quant_space: if abs(c - i) < 1e-3: flag = False break if flag: SGQPackage.__quant_code.append(i) SGQPackage.__quant_codec.set_codec(SGQPackage.__quant_code)
class QuantizedClient(Codec): q_space = build_quantization_space( int(GlobalSettings.get_params(Quantization_Resolution_Client))) def __init__(self, node_id): super().__init__(node_id) def dispose(self): pass def update_blocks( self, block_weight: BlockWeight) -> netEncapsulation[QuantizedPack]: package = QuantizedPack( self.node_id, *Q_g(block_weight.content, QuantizedClient.q_space)) return netEncapsulation(Parameter_Server, package) def receive_blocks(self, package: IPack): self.set_result(package.content, lambda x, y: y)
def update_blocks( self, block_weight: BlockWeight ) -> netEncapsulation[Dict[str, np.ndarray]]: print('Weights delta received.') print('from block: {}'.format(block_weight.block_id)) print('It has a content with shape: {}'.format( block_weight.content.shape)) # 获取没有该数据的节点 send_to = GlobalSettings.get_default().get_adversary( block_weight.block_id) # 我们使用 'data' 字符串来标记我们的梯度内容 pkg = {'data': block_weight.content} # 记录本机梯度 self.__global_weights += block_weight.content self.__current_recv += 1 # 检查是否接受完所有数据 self.__do_grad_average() # 发送梯度 return netEncapsulation(send_to, pkg)
class QuantizedParaServer(Codec): codex = quant( build_quantization_space( int(GlobalSettings.get_params(Quantization_Resolution_Server)))) def __init__(self, node_id): super().__init__(node_id) self.__global_weights: np.ndarray = 0 def dispose(self): pass def update_blocks(self, block_weight: BlockWeight): pass def receive_blocks(self, package: IPack): self.__global_weights -= package.decode(QuantizedClient.codex) reply = QuantizedPack(Parameter_Server, self.__global_weights, QuantizedParaServer.codex) return netEncapsulation(package.node_id, reply)
def record(self, message: str): from parallel_sgd.codec import GlobalSettings GlobalSettings.global_logger().log_message( "Codec: {}, Report: {}.".format(self.__class__.__name__, message))
# const parameters SLAVE_CNT = 4 REDUNDANCY = 1 TEST_ROUNDS = 10 WEIGHTS_SHAPE = np.random.randint(3, 1024, size=2) LAYER = 0 BATCHSIZE = 64 SYNCWAITTIMEOUT = 1000 #ms # setup global parameters GlobalSettings.deprecated_default_settings = DuplicateAssignment( SLAVE_CNT, REDUNDANCY) # default setting Default = GlobalSettings.get_default() class TestCodec(unittest.TestCase): def test_p2p(self): # build codec slave_codec = [SLAVE_CODEC(node_id=i) for i in range(SLAVE_CNT)] for i in range(TEST_ROUNDS): # starting consensus stage node_id = 0 for slave in slave_codec: # build each block for block_id in Default.node_2_block[node_id]: # get random arr = np.random.random(size=WEIGHTS_SHAPE)
def start(self, com: communication.Communication) -> None: state, report = self.__check() self.__log.log_message("Ready:{} \n\t Check List:\n\t\t--> {}".format(state, "\n\t\t--> ".join(report))) # get dataset train_x, train_y, test_x, test_y = self.__data.load() self.__log.log_message('Dataset is ready, type: ({})'.format(self.__data)) # build data feeder block_ids = GlobalSettings.get_default().node_2_block[self.node_id] feeder = PSGDBlockDataFeeder(train_x, train_y, batch_iter=self.__batch_iter, block_ids=block_ids) # assemble optimizer self.__optimizer.assemble(transfer=self.__trans, block_mgr=feeder) # compile model self.__model.compile(self.__optimizer) # summary summary = self.__model.summary() self.__log.log_message(summary) trace_head = '{}-N({})'.format(self.__misc.mission_title, self.node_id) self.__log.log_message('Model set to ready.') log_head = self.__log.Title # start ! GlobalSettings.deprecated_global_logger = self.__log self.__trans.start_transfer(com, group_offset=list(self.group)[0], printer=self.__log, node_id=self.node_id) # record data time_start = time.time() data_send_start = com.bytes_sent data_recv_start = com.bytes_read evaluation_history = [] title = [] r = {} # this message will start the progress reporting com.report_progress(0) # do until reach the target accuracy for i in range(self.__misc.epoch): # change title self.__log.Title = log_head + "-Epo-{}".format(i + 1) history = self.__model.fit(feeder, epoch=1, printer=self.__log) # do tests r = self.__model.evaluate(test_x, test_y) title = r.keys() row = r.values() self.__log.log_message('Evaluate result: {}'.format(r)) evaluation_history.append(row) if self.__misc.target_acc is not None: # only one metric in model metrics list. # evaluation[0] refers to loss # evaluation[1] refers to accuracy. if r[1] > self.__misc.target_acc: break # report progress com.report_progress(int(100 * ((i + 1) / self.__misc.epoch))) # record data time_end = time.time() data_sent_end = com.bytes_sent data_recv_end = com.bytes_read training_history = self.__model.fit_history() # save training history data training_name = "TR-" + trace_head + ".csv" training_trace = pd.DataFrame(training_history.history, columns=training_history.title) training_trace.to_csv(training_name, index=False) # save evaluation history data evaluation_name = "EV-" + trace_head + ".csv" evaluation_trace = pd.DataFrame(evaluation_history, columns=title) evaluation_trace.to_csv(evaluation_name, index=False) # save model model_name = "MODEL-" + trace_head + ".model" self.__model.compile(nn.gradient_descent.SGDOptimizer(learn_rate=1e-5)) self.__model.save(model_name) self.__trace_filename.append(training_name) self.__trace_filename.append(evaluation_name) self.__trace_filename.append(model_name) self.__log.log_message('Execution complete, time: {}.'.format(time_end - time_start)) self.__log.log_message('Execution complete, Total bytes sent: {}.'.format(data_sent_end - data_send_start)) self.__log.log_message('Execution complete, Total bytes read: {}.'.format(data_recv_end - data_recv_start)) self.__log.log_message('Trace file has been saved to {}.'.format(trace_head)) self.__log.flush() # set marker self.__done = True # dispose self.__model.clear() del train_x, train_y, test_x, test_y del self.__model, self.__log # return last evaluation result return r