def test_slot_tagging_f1(self): # (1) Test on oracle frame. slot_tagging_f1_oracle = metrics.get_slot_tagging_f1( self.frame_ref, self.frame_ref, self.utterance, self.schema) # Ground truth values for oracle prediction are all 1.0. self._assert_dicts_almost_equal( {k: 1.0 for k in self.known_metrics[metrics.SLOT_TAGGING_F1]}, slot_tagging_f1_oracle._asdict()) # (2) Test on a previously known frame. slot_tagging_f1_hyp = metrics.get_slot_tagging_f1( self.frame_ref, self.frame_hyp, self.utterance, self.schema) self._assert_dicts_almost_equal( self.known_metrics[metrics.SLOT_TAGGING_F1], slot_tagging_f1_hyp._asdict())
def get_metrics(dataset_ref, dataset_hyp, service_schemas, in_domain_services): """Calculate the DSTC8 metrics. Args: dataset_ref: The ground truth dataset represented as a dict mapping dialogue id to the corresponding dialogue. dataset_hyp: The predictions in the same format as `dataset_ref`. service_schemas: A dict mapping service name to the schema for the service. in_domain_services: The set of services which are present in the training set. Returns: A dict mapping a metric collection name to a dict containing the values for various metrics. Each metric collection aggregates the metrics across a specific set of frames in the dialogues. """ # Metrics can be aggregated in various ways, eg over all dialogues, only for # dialogues containing unseen services or for dialogues corresponding to a # single service. This aggregation is done through metric_collections, which # is a dict mapping a collection name to a dict, which maps a metric to a list # of values for that metric. Each value in this list is the value taken by # the metric on a frame. metric_collections = collections.defaultdict( lambda: collections.defaultdict(list)) # Ensure the dialogs in dataset_hyp also occur in dataset_ref. assert set(dataset_hyp.keys()).issubset(set(dataset_ref.keys())) tf.logging.info("len(dataset_hyp)=%d, len(dataset_ref)=%d", len(dataset_hyp), len(dataset_ref)) # Store metrics for every frame for debugging. per_frame_metric = {} for dial_id, dial_hyp in dataset_hyp.items(): dial_ref = dataset_ref[dial_id] if set(dial_ref["services"]) != set(dial_hyp["services"]): raise ValueError( "Set of services present in ground truth and predictions don't match " "for dialogue with id {}".format(dial_id)) for turn_id, (turn_ref, turn_hyp) in enumerate( zip(dial_ref["turns"], dial_hyp["turns"])): if turn_ref["speaker"] != turn_hyp["speaker"]: raise ValueError( "Speakers don't match in dialogue with id {}".format( dial_id)) # Skip system turns because metrics are only computed for user turns. if turn_ref["speaker"] != "USER": continue if turn_ref["utterance"] != turn_hyp["utterance"]: tf.logging.info("Ref utt: %s", turn_ref["utterance"]) tf.logging.info("Hyp utt: %s", turn_hyp["utterance"]) raise ValueError( "Utterances don't match for dialogue with id {}".format( dial_id)) hyp_frames_by_service = { frame["service"]: frame for frame in turn_hyp["frames"] } # Calculate metrics for each frame in each user turn. for frame_ref in turn_ref["frames"]: service_name = frame_ref["service"] if service_name not in hyp_frames_by_service: raise ValueError( "Frame for service {} not found in dialogue with id {}" .format(service_name, dial_id)) service = service_schemas[service_name] frame_hyp = hyp_frames_by_service[service_name] active_intent_acc = metrics.get_active_intent_accuracy( frame_ref, frame_hyp) slot_tagging_f1_scores = metrics.get_slot_tagging_f1( frame_ref, frame_hyp, turn_ref["utterance"], service) requested_slots_f1_scores = metrics.get_requested_slots_f1( frame_ref, frame_hyp) goal_accuracy_dict = metrics.get_average_and_joint_goal_accuracy( frame_ref, frame_hyp, service) frame_metric = { metrics.ACTIVE_INTENT_ACCURACY: active_intent_acc, metrics.REQUESTED_SLOTS_F1: requested_slots_f1_scores.f1, metrics.REQUESTED_SLOTS_PRECISION: requested_slots_f1_scores.precision, metrics.REQUESTED_SLOTS_RECALL: requested_slots_f1_scores.recall } if slot_tagging_f1_scores is not None: frame_metric[ metrics.SLOT_TAGGING_F1] = slot_tagging_f1_scores.f1 frame_metric[metrics.SLOT_TAGGING_PRECISION] = ( slot_tagging_f1_scores.precision) frame_metric[ metrics. SLOT_TAGGING_RECALL] = slot_tagging_f1_scores.recall frame_metric.update(goal_accuracy_dict) frame_id = "{:s}-{:03d}-{:s}".format(dial_id, turn_id, frame_hyp["service"]) per_frame_metric[frame_id] = frame_metric # Add the frame-level metric result back to dialogues. frame_hyp["metrics"] = frame_metric # Get the domain name of the service. domain_name = frame_hyp["service"].split("_")[0] domain_keys = [ALL_SERVICES, frame_hyp["service"], domain_name] if frame_hyp["service"] in in_domain_services: domain_keys.append(SEEN_SERVICES) else: domain_keys.append(UNSEEN_SERVICES) for domain_key in domain_keys: for metric_key, metric_value in frame_metric.items(): if metric_value != metrics.NAN_VAL: metric_collections[domain_key][metric_key].append( metric_value) all_metric_aggregate = {} for domain_key, domain_metric_vals in metric_collections.items(): domain_metric_aggregate = {} for metric_key, value_list in domain_metric_vals.items(): if value_list: # Metrics are macro-averaged across all frames. domain_metric_aggregate[metric_key] = float( np.mean(value_list)) else: domain_metric_aggregate[metric_key] = metrics.NAN_VAL all_metric_aggregate[domain_key] = domain_metric_aggregate return all_metric_aggregate, per_frame_metric