def evaluate_tflite(self, tflite_filepath, data): """Evaluates the tflite model. Args: tflite_filepath: File path to the TFLite model. data: Data to be evaluated. Returns: The evaluation result of TFLite model - accuracy. """ ds = data.gen_dataset( batch_size=1, is_training=False, preprocess=self.preprocess) predictions, labels = [], [] lite_runner = model_util.get_lite_runner(tflite_filepath, self.model_spec) for i, (feature, label) in enumerate(data_util.generate_elements(ds)): log_steps = 1000 tf.compat.v1.logging.log_every_n(tf.compat.v1.logging.INFO, 'Processing example: #%d\n%s', log_steps, i, feature) probabilities = lite_runner.run(feature) predictions.append(np.argmax(probabilities)) # Gets the ground-truth labels. label = label[0] if label.size > 1: # one-hot tensor. label = np.argmax(label) labels.append(label) predictions, labels = np.array(predictions), np.array(labels) result = {'accuracy': (predictions == labels).mean()} return result
def evaluate_tflite(self, tflite_filepath, data): """Evaluates the tflite model. The data is padded to required length, and multiple metrics are evaluated. Args: tflite_filepath: File path to the TFLite model. data: Data to be evaluated. Returns: Dict of (metric, value), evaluation result of TFLite model. """ lite_runner = model_util.get_lite_runner(tflite_filepath, self.model_spec) ds = data.gen_dataset(batch_size=1, is_training=False) max_output_size = data.max_vocab_id + 1 # +1 because 0 is reserved for OOV. eval_top_k = self.model_spec.params['eval_top_k'] metrics = [ _metrics.GlobalRecall(name=f'Global_Recall/Recall_{k}', top_k=k) for k in eval_top_k ] for feature, label in data_util.generate_elements(ds): context = feature['context'][0].astype(np.int32) # x: shape [max_length], padded with PAD_ID if len(context) == self._max_history_length: x = context elif len(context) > self._max_history_length: x = context[:self._max_history_length] else: # Padding. x = np.empty([self._max_history_length], dtype=np.int32) x.fill(self.OOV_ID) x[:len(context)] = context # ids, scores: shape [top_k] ids, scores = lite_runner.run(x) # y_true: shape [1, 1] y_true = label # y_pred: shape [1, max_output_size]; fill only scores with top-k ids. y_pred = np.zeros([1, max_output_size]) for i, score in zip(ids, scores): if i in data.vocab: # Only set if id is in vocab. y_pred[0, i] = score # Update metrics. for m in metrics: m.update_state(y_true, y_pred) result = collections.OrderedDict([(m.name, m.result()) for m in metrics]) return result
def evaluate_tflite(self, tflite_filepath, data): """Evaluates the tflite model. The data is padded to required length, and multiple metrics are evaluated. Args: tflite_filepath: File path to the TFLite model. data: Data to be evaluated. Returns: Dict of (metric, value), evaluation result of TFLite model. """ label_name = self.input_spec.label_feature.feature_name lite_runner = model_util.get_lite_runner(tflite_filepath, self.model_spec) ds = data.gen_dataset(batch_size=1, is_training=False) max_output_size = data.max_vocab_id + 1 # +1 because 0 is reserved for OOV. eval_top_k = self.model_hparams.eval_top_k metrics = [ _metrics.GlobalRecall(top_k=k, name=f'Global_Recall/Recall_{k}') for k in eval_top_k ] for feature, y_true in data_util.generate_elements(ds): feature.pop(label_name) x = feature ids, scores = lite_runner.run(x) # y_true: shape [1, 1] # y_pred: shape [1, max_output_size]; fill only scores with top-k ids. y_pred = np.zeros([1, max_output_size]) for i, score in zip(ids, scores): if i in data.vocab: # Only set if id is in vocab. y_pred[0, i] = score # Update metrics. for m in metrics: m.update_state(y_true, y_pred) result = collections.OrderedDict([(m.name, m.result()) for m in metrics]) return result
def evaluate_tflite(self, tflite_filepath, data): ds = data.gen_dataset(batch_size=1, is_training=False) predictions, labels = [], [] lite_runner = model_util.get_lite_runner(tflite_filepath, self.model_spec) for i, (feature, label) in enumerate(data_util.generate_elements(ds)): log_steps = 1000 tf.compat.v1.logging.log_every_n(tf.compat.v1.logging.INFO, 'Processing example: #%d\n%s', log_steps, i, feature) probability = lite_runner.run(feature) # Shape: (batch=1, 1) probability = probability.flatten()[0] # Get sclar value predictions.append(probability > 0.5) label = label[0] labels.append(label) predictions = np.array(predictions).astype(int) labels = np.array(labels).astype(int) return {'accuracy': (predictions == labels).mean()}