def report_evaluation_metrics(self, model_outputs_pb, labels): labels = tensor_pb_to_ndarray(labels) model_outputs = {} for tensor_pb in model_outputs_pb: key = tensor_pb.name model_outputs[key] = tensor_pb_to_ndarray(tensor_pb) self.update_evaluation_metrics(model_outputs, labels)
def report_evaluation_metrics(self, model_outputs, labels): labels = tensor_pb_to_ndarray(labels) for tensor_pb in model_outputs: key = tensor_pb.name metrics = self._metrics_dict.get(key, {}) if not metrics: continue outputs = tensor_pb_to_ndarray(tensor_pb) for metric_inst in metrics.values(): metric_inst.update_state(labels, outputs)
def pull_embedding_vector(self, layer_name, embedding_ids): """Pulls and returns embedding vectors ordered by the embedding ids.""" ps_ids = {} ps_ids_index = {} for idx, embedding_id in enumerate(embedding_ids): ps_id = int_to_id(embedding_id, self._ps_num) ps_ids.setdefault(ps_id, []).append(embedding_id) ps_ids_index.setdefault(ps_id, []).append(idx) embeddings = [] index = [] pb_future_and_id_pairs = [] for ps_id, embedding_ids in ps_ids.items(): req = elasticdl_pb2.PullEmbeddingVectorRequest() req.name = layer_name req.ids.extend(embedding_ids) pb_future = self._ps_stubs[ps_id].pull_embedding_vector.future(req) pb_future_and_id_pairs.append((pb_future, ps_id)) for pb_future, ps_id in pb_future_and_id_pairs: pb = pb_future.result() embeddings.append(tensor_pb_to_ndarray(pb)) index.extend(ps_ids_index[ps_id]) embeddings = np.concatenate(embeddings) # adjust the order of embedding vectors new_embeddings = np.empty_like(embeddings) new_embeddings[index] = embeddings return new_embeddings
def test_pull_variable(self): self.create_default_server_and_stub() param0 = { "v0": np.random.rand(3, 2).astype(np.float32), "v1": np.random.rand(10, 32).astype(np.float32), } pull_req = empty_pb2.Empty() # try to pull variable res = self._stub.pull_variable(pull_req) # not initialized self.assertFalse(res.model_init_status) # init variable req = elasticdl_pb2.Model() req.version = 1 for name, var in param0.items(): emplace_tensor_pb_from_ndarray(req.param, var, name=name) res = self._stub.push_model(req) self.assertEqual(res, empty_pb2.Empty()) # pull variable back res = self._stub.pull_variable(pull_req) self.assertTrue(res.model_init_status) self.assertEqual(res.model.version, req.version) for param in res.model.param: name = param.name tensor = tensor_pb_to_ndarray(param) self.assertTrue(np.allclose(param0[name], tensor))
def _check_get_model_response(self, version, expected, response): self.assertEqual(version, response.version) self.assertEqual( list(sorted(expected.keys())), sorted([v.name for v in response.param]), ) for var in response.param: exp_value = expected[var.name] np.testing.assert_array_equal(exp_value, tensor_pb_to_ndarray(var))
def get_embedding_vectors(self, name, ids): pull_req = elasticdl_pb2.PullEmbeddingVectorRequest() pull_req.name = name pull_req.ids.extend(ids) res = self._stub.pull_embedding_vector(pull_req) if res.content: return tensor_pb_to_ndarray(res) else: return None
def report_evaluation_metrics(self, evaluation_version, model_outputs, labels): if (self.model_version >= 0 and evaluation_version >= 0 and evaluation_version != self.model_version): logger.error( "Drop a wrong version evaluation: request %d, receive %d" % (self.model_version, evaluation_version)) return False labels = tensor_pb_to_ndarray(labels) for tensor_pb in model_outputs: key = tensor_pb.name metrics = self._metrics_dict.get(key, {}) if not metrics: continue outputs = tensor_pb_to_ndarray(tensor_pb) for metric_inst in metrics.values(): metric_inst.update_state(labels, outputs) return True
def test_tensor_pb_to_ndarray(self): values = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]], np.float32) indices = np.array([0, 2]) pb = self._create_tensor_pb(values) self.assertTrue(np.allclose(tensor_pb_to_ndarray(pb), values)) # convert a tensor_pb with sparse tensor to a ndarray, should raise pb = self._create_tensor_pb(values, indices) self.assertRaises(NotImplementedError, tensor_pb_to_ndarray, pb)
def _init_non_embedding_params(self, tensors_pb): for pb in tensors_pb: name = pb.name arr = tensor_pb_to_ndarray(pb) # Please note that `tf.Variable` will do something with magic. # If you pass a name "somename" to a `tf.Variable`, the final # variable name will be "somename:0". So the `tf.Variable.name` # is meaningless, we must avoid use it in PS side. var = tf.Variable(initial_value=arr, trainable=True) self.non_embedding_params[name] = var
def _restore_params_from_pb(self, tensors_pb): for pb in tensors_pb: name = pb.name if not pb.indices: # Please note that `tf.Variable` will do something with magic. # If you pass a name "somename" to a `tf.Variable`, the final # variable name will be "somename:0". So the `tf.Variable.name` # is meaningless, we must avoid use it in PS side. arr = tensor_pb_to_ndarray(pb) var = tf.Variable(initial_value=arr, trainable=True) self.non_embedding_params[name] = var else: # Only pb of embedding parameters has indices. tensor = Tensor() deserialize_tensor_pb(pb, tensor) self.embedding_params[name].set(tensor.indices, tensor.values)
def _init_model_from_tensor_pb_list(self, tensor_pb_list): assert tensor_pb_list for pb in tensor_pb_list: self.set_model_var(pb.name, tensor_pb_to_ndarray(pb))