def handle_commit(self, conn, addr): # Receive the parameters from the remote node. data = recv_data(conn) # Extract the data from the dictionary. r = np.multiply( np.negative(np.asarray(data['residual'])), self.worker_learning_rate_inverse ) # Convert residuals to gradient, divides by SGD learning rate in worker level assert r.shape == self.center_variable.shape # Assert length of gradients given is equal to size of weight parameters with self.mutex: # Update variables self.t += 1 # Increase timestep self.m *= self.b1 self.m += np.multiply( r, 1 - self.b1) # Update biased first moment estimate self.v *= self.b2 self.v += np.multiply(np.power( r, 2), 1 - self.b2) # Update biased second moment estimate self.m_norm = np.multiply(self.m, np.divide( 1, 1 - self.b1** self.t)) # Compute bias-corrected first moment estimate self.v_norm = np.multiply(self.v, np.divide( 1, 1 - self.b2** self.t)) # Compute bias-corrected second moment estimate self.center_variable -= np.multiply( np.divide(self.m_norm, np.power(self.v_norm, 0.5) + self.e), self.a) # Update parameters # Increment the number of parameter server updates. self.next_update()
def pull(self): """Fetches the center variable from the parameter server.""" # Request a pull from the parameter server. self.socket.sendall(b'p') # Fetch the central variable from the parameter server. center_variable = recv_data(self.socket) self.center_variable = np.asarray(center_variable)
def pull(self): """Requests the center variable and last update from the parameter server.""" # Request a pull from the parameter server. self.socket.sendall(b'p') # Fetch the dictionary from the parameter server. data = recv_data(self.socket) self.center_variable = np.asarray(data['model']) self.last_update = data['update']
def handle_commit(self, conn, addr): # Receive the parameters from the remote node. data = recv_data(conn) # Extract the data from the dictionary. r = data['residual'] with self.mutex: # Update the center variable. self.center_variable = self.center_variable + r # Increment the number of parameter server updates. self.next_update()
def handle_commit(self, conn, addr): # Receive the parameters from the remote node. data = recv_data(conn) # Extract the delta from the dictionary. delta = data['delta'] # Update the center variable with the delta. with self.mutex: self.center_variable = self.center_variable + delta # Next iteration. self.next_update()
def handle_commit(self, conn, addr): data = recv_data(conn) r = data['residual'] # Fetch the last iteration number last_update = data['last_update'] du = (self.num_updates - last_update) + 1 r /= du with self.mutex: center_variable = self.model.get_weights() center_variable = center_variable + r self.model.set_weights(center_variable) # Increment the number of parameter server updates. self.next_update()
def handle_commit(self, conn, addr): # Receive the parameters from the remote node. data = recv_data(conn) # Extract the delta from the dictionary. delta = data['delta'] # Update the center variable with the delta. with self.mutex: # Fetch the center variable. center_variable = self.model.get_weights() center_variable = center_variable + delta # Set the new parameters of the model. self.model.set_weights(center_variable) # Next iteration. self.next_update()
def handle_commit(self, conn, addr): data = recv_data(conn) # Check if the data contains end of a process. if "worker_done" in data: self.num_workers -= 1 else: # Fetch the residual. r = data['residual'] with self.mutex: center_variable = self.model.get_weights() center_variable = center_variable + r self.model.set_weights(center_variable) # Increment the number of parameter server updates. self.next_update()
def pull(self): #establish the connection if self.socket_parent is None: self.connect() """Requests the center variable from the parameter server.""" # Request a pull from the parameter server. self.setup_pull() # Fetch the center variable from the parent parameter server. with self.in_mutex: temp = np.asarray(recv_data(self.socket_parent)) with self.mutex: #add the culmulated commit from children self.center_variable = temp + self.center_variable - self.center_variable_old self.center_variable_old = temp
def handle_commit(self, conn, addr): # Receive the parameters from the remote node. data = recv_data(conn) # Extract the data from the dictionary. r = data['residual'] worker_id = data['worker_id'] stale_cv = data['stale_center_variable'] with self.mutex: diff_cv = np.subtract(self.center_variable, stale_cv) d = 1 / (self.inverse_learning_rate * np.power(diff_cv, 2) + 1) r = np.multiply(d, r) # Update the center variable. self.center_variable = self.center_variable + r # Increment the number of parameter server updates. self.next_update()
def handle_commit(self, conn, addr): # Receive the parameters from the remote node. data = np.array_split(np.asarray(recv_data(conn)['residual']), self.processes) with self.mutex: # Update variables self.t += 1 # Increase timestep result = [ self.pool.apply(pooling_function, args=(data[i], self.center_variable[i], self.m[i], self.v[i], self.a, self.b1, self.b2, self.e, self.t, self.worker_learning_rate_inverse)) for i in range(self.processes) ] for i in range(len(result)): self.center_variable[i], self.m[i], self.v[i] = result[i][ 0], result[i][1], result[i][2] # Increment the number of parameter server updates. self.next_update()
def pull(self): """Requests the center variable from the parameter server.""" # Request a pull from the parameter server. self.socket.sendall(b'p') # Fetch the center variable from the parameter server. self.center_variable = np.asarray(recv_data(self.socket))
def pull(self): """Requests the center variable from the parameter server.""" # Request a pull from the parameter server. self.setup_pull() # Fetch the center variable from the parameter server. self.center_variable = np.asarray(recv_data(self.socket))