def report_gradient(self, grads): """ report gradient to ps, return (accepted, model_version) from rpc call. """ req = elasticdl_pb2.ReportGradientRequest() origin_vars = self._model.trainable_variables origin_var_n = len(origin_vars) # should keep the same order as self.get_trainable_items() for g, v in zip(grads[:origin_var_n], origin_vars): if isinstance(g, tf.IndexedSlices): req.gradient[v.name].CopyFrom( ndarray_to_tensor( g.values.numpy(), tuple(g.indices.numpy()) ) ) else: req.gradient[v.name].CopyFrom(ndarray_to_tensor(g.numpy())) # deal with gradients of ElasticDL embedding layer # should keep the same order as self.get_trainable_items() if self._embedding_layers: grads_edlembedding = grads[origin_var_n:] bet_number = 0 for layer in self._embedding_layers: bet_number += len(layer.bet_ids_pair) if len(grads_edlembedding) != bet_number: raise ValueError( "elasticdl.layers.embedding related gradient number %d " "does not match the number of its output tensor %d." % (len(grads_edlembedding), bet_number) ) it = 0 for layer in self._embedding_layers: g_values = None g_indices = None for bet, ids in layer.bet_ids_pair: grad = grads_edlembedding[it] it += 1 # ElasticDL embedding layer with Sparse Gradients if isinstance(grad, tf.IndexedSlices): grad = grad.values if g_values is not None: g_values = tf.concat([g_values, grad], axis=0) g_indices = tf.concat([g_indices, ids], axis=0) else: g_values = grad g_indices = ids req.gradient[layer.name].CopyFrom( ndarray_to_tensor( g_values.numpy(), tuple(g_indices.numpy()) ) ) req.model_version = self._model_version res = self._stub.ReportGradient(req) return res.accepted, res.model_version
def report_gradient(self, grads): """ report gradient to ps, return (accepted, model_version) from rpc call. """ req = elasticdl_pb2.ReportGradientRequest() non_embed_vars_n = len(self._non_embed_vars) # The first `non_embed_vars_n` items in `grads` are gradients for # `self._non_embed_vars` for g, v in zip(grads[:non_embed_vars_n], self._non_embed_vars): if isinstance(g, tf.IndexedSlices): req.gradient[v.name].CopyFrom( ndarray_to_tensor(g.values.numpy(), tuple(g.indices.numpy()))) else: req.gradient[v.name].CopyFrom(ndarray_to_tensor(g.numpy())) # Accumulate gradients of ElasticDL embedding layer if self._embedding_layers: # The `edl_embedding_grads` are gradients for bets in # `self._embedding_layers` edl_embedding_grads = grads[non_embed_vars_n:] # Check that the number of bet equal to the number of gradients. # Please note that every embedding layer may have more than one # `bet_id_pair`. bet_number = 0 for layer in self._embedding_layers: bet_number += len(layer.embedding_and_ids) if len(edl_embedding_grads) != bet_number: raise ValueError( "elasticdl.layers.embedding related gradient number %d " "does not match the number of its output tensor %d." % (len(edl_embedding_grads), bet_number)) grad_accum_iter = 0 for layer in self._embedding_layers: g_values = None g_indices = None for _, ids in layer.embedding_and_ids: grad = edl_embedding_grads[grad_accum_iter] grad_accum_iter += 1 # ElasticDL embedding layer with Sparse Gradients if isinstance(grad, tf.IndexedSlices): grad = grad.values if g_values is not None: g_values = tf.concat([g_values, grad], axis=0) g_indices = tf.concat([g_indices, ids], axis=0) else: g_values = grad g_indices = ids req.gradient[layer.name].CopyFrom( ndarray_to_tensor(g_values.numpy(), tuple(g_indices.numpy()))) req.model_version = self._model_version res = self._stub.ReportGradient(req) return res.accepted, res.model_version
def makeGrad(): """ Make a ReportGradientRequest compatible with model""" req = elasticdl_pb2.ReportGradientRequest() req.gradient["x"].CopyFrom( ndarray_to_tensor(np.array([0.1], dtype=np.float32))) req.gradient["y"].CopyFrom( ndarray_to_tensor(np.array([0.03, 0.06], dtype=np.float32))) req.model_version = 1 return req
def makeGrad(): """ Make a ReportGradientRequest compatible with model""" req = elasticdl_pb2.ReportGradientRequest() emplace_tensor_pb_from_ndarray(req.gradient, np.array([0.1], np.float32), name="x") emplace_tensor_pb_from_ndarray(req.gradient, np.array([0.03, 0.06], np.float32), name="y") req.model_version = 1 return req