def push_update(self, update):
print("worker {} push update >>> id = {}, n_iter = {}".format(
update.worker_id, update.id, update.n_iter))
if self.models.__contains__(update.id):
save_start = time.time()
f_name = "{}/{}_{}_{}.npy".format(
self.tmp_dir, update.id, update.worker_id,
self.model_parallelism[update.id])
np.save(f_name, np.array(update.data))
print("save file {}, cost {} s".format(f_name,
time.time() - save_start))
self.inc_num_file(update.id)
if self.can_merge(update.id):
merge_start = time.time()
self.merge_update(update.id)
print("merge cost {} s".format(time.time() - merge_start))
self.w_lock.acquire()
self.model_push_count[
update.id] = self.model_push_count[update.id] + 1
self.w_lock.release()
else:
x = InvalidOperation()
x.whatOp = Operation.PUSH_UPDATE
x.why = 'No model {} on PS'.format(update.id)
raise x
def can_push(self, mid, n_iter, worker_id):
#print("worker {} ask can_push >>> id = {}, iter = {}".format(worker_id, mid, n_iter))
if self.model_pull_count.__contains__(mid):
return self.model_pull_count[
mid] == self.model_parallelism[mid] * (n_iter + 1)
else:
x = InvalidOperation()
x.whatOp = Operation.CAN_PUSH
x.why = 'No model {} in model_pull_count on PS'.format(mid)
raise x
def pull_model(self, mid, n_iter, worker_id):
print("worker {} pull model >>> id = {}, iter = {}".format(
worker_id, mid, n_iter))
if self.models.__contains__(mid):
model = Model()
model.id = mid
model.data = self.models[mid].tolist()
model.length = len(model.data)
self.model_pull_count[mid] = self.model_pull_count[mid] + 1
return model
else:
x = InvalidOperation()
x.whatOp = Operation.PULL_MODEL
x.why = 'No model {} on PS'.format(mid)
raise x
def push_grad(self, grad):
print("worker {} push grad >>> id = {}, lr = {}, n_iter = {}".format(
grad.worker_id, grad.id, grad.learning_rate, grad.n_iter))
if self.models.__contains__(grad.id):
self.tmp_data.append(np.array(grad.data))
self.inc_num_data(grad.id)
if self.can_merge(grad.id):
self.merge_grad(grad.id, grad.learning_rate)
self.w_lock.acquire()
self.model_push_count[grad.id] = self.model_push_count[grad.id] + 1
self.w_lock.release()
else:
x = InvalidOperation()
x.whatOp = Operation.PUSH_GRAD
x.why = 'No model {} on PS'.format(grad.id)
raise x
def push_grad(self, grad):
print("worker {} push grad >>> id = {}, lr = {}, n_iter = {}".format(
grad.worker_id, grad.id, grad.learning_rate, grad.n_iter))
update_start = time.time()
if self.models.__contains__(grad.id):
self.w_lock.acquire()
self.models[grad.id] = np.add(
self.models.get(grad.id),
np.multiply(grad.data, grad.learning_rate))
self.model_push_count[grad.id] = self.model_push_count[grad.id] + 1
self.w_lock.release()
else:
x = InvalidOperation()
x.whatOp = Operation.PUSH_GRAD
x.why = 'No model {} on PS'.format(grad.id)
raise x
print("update model cost {} s".format(time.time() - update_start))
def push_update(self, update):
print("worker {} push update >>> id = {}, n_iter = {}".format(
update.worker_id, update.id, update.n_iter))
if self.models.__contains__(update.id):
self.tmp_data.append(np.array(update.data))
self.inc_num_data(update.id)
if self.can_merge(update.id):
self.merge_update(update.id)
self.w_lock.acquire()
self.model_push_count[
update.id] = self.model_push_count[update.id] + 1
self.w_lock.release()
else:
x = InvalidOperation()
x.whatOp = Operation.PUSH_UPDATE
x.why = 'No model {} on PS'.format(update.id)
raise x
def push_update(self, update):
print("worker {} push update >>> id = {}, n_iter = {}".format(
update.worker_id, update.id, update.n_iter))
update_start = time.time()
if self.models.__contains__(update.id):
self.w_lock.acquire()
self.models[update.id] = np.add(self.models.get(update.id),
update.data)
self.model_push_count[
update.id] = self.model_push_count[update.id] + 1
self.w_lock.release()
else:
x = InvalidOperation()
x.whatOp = Operation.PUSH_UPDATE
x.why = 'No model {} on PS'.format(update.id)
raise x
print("update model cost {} s".format(time.time() - update_start))
def push_grad(self, grad):
print("worker {} push grad >>> id = {}, lr = {}, n_iter = {}".format(
grad.worker_id, grad.id, grad.learning_rate, grad.n_iter))
if self.models.__contains__(grad.id):
save_start = time.time()
f_name = "{}/{}_{}_{}.npy".format(self.tmp_dir, grad.id,
grad.worker_id,
self.model_parallelism[grad.id])
np.save(f_name, np.array(grad.data))
print("save file {}, cost {} s".format(f_name,
time.time() - save_start))
self.inc_num_file(grad.id)
if self.can_merge(grad.id):
merge_start = time.time()
self.merge_grad(grad.id, grad.learning_rate)
print("merge cost {} s".format(time.time() - merge_start))
self.w_lock.acquire()
self.model_push_count[grad.id] = self.model_push_count[grad.id] + 1
self.w_lock.release()
else:
x = InvalidOperation()
x.whatOp = Operation.PUSH_GRAD
x.why = 'No model {} on PS'.format(grad.id)
raise x