def handle_commit(self, conn, addr):
# Receive the parameters from the remote node.
data = recv_data(conn)
# Extract the data from the dictionary.
r = np.multiply(
np.negative(np.asarray(data['residual'])),
self.worker_learning_rate_inverse
) # Convert residuals to gradient, divides by SGD learning rate in worker level
assert r.shape == self.center_variable.shape # Assert length of gradients given is equal to size of weight parameters
with self.mutex:
# Update variables
self.t += 1 # Increase timestep
self.m *= self.b1
self.m += np.multiply(
r, 1 - self.b1) # Update biased first moment estimate
self.v *= self.b2
self.v += np.multiply(np.power(
r, 2), 1 - self.b2) # Update biased second moment estimate
self.m_norm = np.multiply(self.m, np.divide(
1, 1 - self.b1**
self.t)) # Compute bias-corrected first moment estimate
self.v_norm = np.multiply(self.v, np.divide(
1, 1 - self.b2**
self.t)) # Compute bias-corrected second moment estimate
self.center_variable -= np.multiply(
np.divide(self.m_norm,
np.power(self.v_norm, 0.5) + self.e),
self.a) # Update parameters
# Increment the number of parameter server updates.
self.next_update()
def pull(self):
"""Fetches the center variable from the parameter server."""
# Request a pull from the parameter server.
self.socket.sendall(b'p')
# Fetch the central variable from the parameter server.
center_variable = recv_data(self.socket)
self.center_variable = np.asarray(center_variable)
def pull(self):
"""Requests the center variable and last update from the parameter server."""
# Request a pull from the parameter server.
self.socket.sendall(b'p')
# Fetch the dictionary from the parameter server.
data = recv_data(self.socket)
self.center_variable = np.asarray(data['model'])
self.last_update = data['update']
def handle_commit(self, conn, addr):
# Receive the parameters from the remote node.
data = recv_data(conn)
# Extract the data from the dictionary.
r = data['residual']
with self.mutex:
# Update the center variable.
self.center_variable = self.center_variable + r
# Increment the number of parameter server updates.
self.next_update()
def handle_commit(self, conn, addr):
# Receive the parameters from the remote node.
data = recv_data(conn)
# Extract the delta from the dictionary.
delta = data['delta']
# Update the center variable with the delta.
with self.mutex:
self.center_variable = self.center_variable + delta
# Next iteration.
self.next_update()
def handle_commit(self, conn, addr):
data = recv_data(conn)
r = data['residual']
# Fetch the last iteration number
last_update = data['last_update']
du = (self.num_updates - last_update) + 1
r /= du
with self.mutex:
center_variable = self.model.get_weights()
center_variable = center_variable + r
self.model.set_weights(center_variable)
# Increment the number of parameter server updates.
self.next_update()
def handle_commit(self, conn, addr):
# Receive the parameters from the remote node.
data = recv_data(conn)
# Extract the delta from the dictionary.
delta = data['delta']
# Update the center variable with the delta.
with self.mutex:
# Fetch the center variable.
center_variable = self.model.get_weights()
center_variable = center_variable + delta
# Set the new parameters of the model.
self.model.set_weights(center_variable)
# Next iteration.
self.next_update()
def handle_commit(self, conn, addr):
data = recv_data(conn)
# Check if the data contains end of a process.
if "worker_done" in data:
self.num_workers -= 1
else:
# Fetch the residual.
r = data['residual']
with self.mutex:
center_variable = self.model.get_weights()
center_variable = center_variable + r
self.model.set_weights(center_variable)
# Increment the number of parameter server updates.
self.next_update()
def pull(self):
#establish the connection
if self.socket_parent is None:
self.connect()
"""Requests the center variable from the parameter server."""
# Request a pull from the parameter server.
self.setup_pull()
# Fetch the center variable from the parent parameter server.
with self.in_mutex:
temp = np.asarray(recv_data(self.socket_parent))
with self.mutex:
#add the culmulated commit from children
self.center_variable = temp + self.center_variable - self.center_variable_old
self.center_variable_old = temp
def handle_commit(self, conn, addr):
# Receive the parameters from the remote node.
data = recv_data(conn)
# Extract the data from the dictionary.
r = data['residual']
worker_id = data['worker_id']
stale_cv = data['stale_center_variable']
with self.mutex:
diff_cv = np.subtract(self.center_variable, stale_cv)
d = 1 / (self.inverse_learning_rate * np.power(diff_cv, 2) + 1)
r = np.multiply(d, r)
# Update the center variable.
self.center_variable = self.center_variable + r
# Increment the number of parameter server updates.
self.next_update()
def handle_commit(self, conn, addr):
# Receive the parameters from the remote node.
data = np.array_split(np.asarray(recv_data(conn)['residual']),
self.processes)
with self.mutex:
# Update variables
self.t += 1 # Increase timestep
result = [
self.pool.apply(pooling_function,
args=(data[i], self.center_variable[i],
self.m[i], self.v[i], self.a, self.b1,
self.b2, self.e, self.t,
self.worker_learning_rate_inverse))
for i in range(self.processes)
]
for i in range(len(result)):
self.center_variable[i], self.m[i], self.v[i] = result[i][
0], result[i][1], result[i][2]
# Increment the number of parameter server updates.
self.next_update()
def pull(self):
"""Requests the center variable from the parameter server."""
# Request a pull from the parameter server.
self.socket.sendall(b'p')
# Fetch the center variable from the parameter server.
self.center_variable = np.asarray(recv_data(self.socket))
def pull(self):
"""Requests the center variable from the parameter server."""
# Request a pull from the parameter server.
self.setup_pull()
# Fetch the center variable from the parameter server.
self.center_variable = np.asarray(recv_data(self.socket))
