def get_scores( sources: PathOrPathsOrDictOfStrList, references: PathOrPathsOrDictOfStrList, model_to_hypotheses: PathOrPathsOrDictOfStrList, metrics: List[str], tags: Optional[PathOrPathsOrDictOfStrList] = None, verbose: bool = False, problem: str = None, ) -> Tuple[Dict, Dict]: # Copyright (c) Facebook, Inc. and its affiliates. # The code in this function is licensed under the MIT license. _srcs = VizSeqDataSources(sources) _refs = VizSeqDataSources(references) _hypos = VizSeqDataSources(model_to_hypotheses) _tags, tag_set = None, [] if tags is not None: _tags = VizSeqDataSources(tags, text_merged=True) tag_set = sorted(_tags.unique()) _tags = _tags.text models = _hypos.names all_metrics = get_scorer_ids() _metrics = [] for s in metrics: if s in all_metrics: _metrics.append(s) else: logger.warning(f'"{s}" is not a valid metric.') def scorer_kwargs(s): kwargs = {"corpus_level": True, "sent_level": False, "verbose": verbose} if s in ( "kendall_task_ranking", "req_cov", "essential_req_cov", "achievement", "granularity", ): # ProcGenScorer's kwargs["extra_args"] = {"problem": problem} return kwargs scores = { s: { m: get_scorer(s)(**scorer_kwargs(s)).score( _hypos.data[i].text, _refs.text, tags=_tags, sources=_srcs.text ) for i, m in enumerate(models) } for s in _metrics } corpus_scores = { s: {m: scores[s][m].corpus_score for m in models} for s in _metrics } group_scores = { s: {t: {m: scores[s][m].group_scores[t] for m in models} for t in tag_set} for s in _metrics } return corpus_scores, group_scores
def view_scores(references: PathOrPathsOrDictOfStrList, hypothesis: Optional[PathOrPathsOrDictOfStrList], metrics: List[str], tags: Optional[PathOrPathsOrDictOfStrList] = None): _ref = VizSeqDataSources(references) _hypo = VizSeqDataSources(hypothesis) _tags, tag_set = None, [] if tags is not None: _tags = VizSeqDataSources(tags, text_merged=True) tag_set = sorted(_tags.unique()) _tags = _tags.text models = _hypo.names all_metrics = get_scorer_ids() _metrics = [] for s in metrics: if s in all_metrics: _metrics.append(s) else: logger.warn(f'"{s}" is not a valid metric.') scores = { s: { m: get_scorer(s)(corpus_level=True, sent_level=False).score(_hypo.data[i].text, _ref.text, tags=_tags) for i, m in enumerate(models) } for s in _metrics } corpus_scores = { s: {m: scores[s][m].corpus_score for m in models} for s in _metrics } group_scores = { s: { t: {m: scores[s][m].group_scores[t] for m in models} for t in tag_set } for s in _metrics } metrics_and_names = [[s, get_scorer_name(s)] for s in _metrics] html = env.get_template('ipynb_scores.html').render( metrics_and_names=metrics_and_names, models=models, tag_set=tag_set, corpus_scores=corpus_scores, group_scores=group_scores, corpus_and_group_score_latex=VizSeqWebView.latex_corpus_group_scores( corpus_scores, group_scores), corpus_and_group_score_csv=VizSeqWebView.csv_corpus_group_scores( corpus_scores, group_scores), ) return HTML(html)
def get( cls, src: VizSeqDataSources, ref: VizSeqDataSources, hypo: VizSeqDataSources, page_sz: int, page_no: int, metrics: Optional[List[str]] = None, query: str = '', sorting: int = 0, sorting_metric: str = '', need_lang_tags: bool = False, disable_alignment: bool = False, ) -> VizSeqPageData: assert page_no > 0 and page_sz > 0 page_sz = min(page_sz, MAX_PAGE_SZ) metrics = [] if metrics is None else metrics models = hypo.text_names # query cur_idx = list(range(len(src))) if src.has_text: cur_idx = VizSeqFilter.filter(src.text, query) elif ref.has_text: cur_idx = VizSeqFilter.filter(ref.text, query) n_samples = len(cur_idx) # sorting sorting = {e.value: e for e in VizSeqSortingType}.get(sorting, None) assert sorting is not None if sorting == VizSeqSortingType.random: cur_idx = VizSeqRandomSorter.sort(cur_idx) elif sorting == VizSeqSortingType.ref_len: cur_idx = VizSeqByLenSorter.sort(ref.main_text, cur_idx) elif sorting == VizSeqSortingType.ref_alphabetical: cur_idx = VizSeqByStrOrderSorter.sort(ref.main_text, cur_idx) elif sorting == VizSeqSortingType.src_len: if src.has_text: cur_idx = VizSeqByLenSorter.sort(src.main_text, cur_idx) elif sorting == VizSeqSortingType.src_alphabetical: if src.has_text: cur_idx = VizSeqByStrOrderSorter.sort(src.main_text, cur_idx) elif sorting == VizSeqSortingType.metric: if sorting_metric in get_scorer_ids(): _cur_ref = [_select(t, cur_idx) for t in ref.text] scores = { m: get_scorer(sorting_metric)(corpus_level=False, sent_level=True).score( _select(t, cur_idx), _cur_ref).sent_scores for m, t in zip(models, hypo.text) } scores = [{m: scores[m][i] for m in models} for i in range(len(cur_idx))] cur_idx = VizSeqByMetricSorter.sort(scores, cur_idx) # pagination start_idx, end_idx = _get_start_end_idx(len(cur_idx), page_sz, page_no) cur_idx = cur_idx[start_idx:end_idx + 1] n_cur_samples = len(cur_idx) # page data cur_src = src.cached(cur_idx) cur_src_text = _select(src.main_text, cur_idx) if src.has_text else None cur_ref = [_select(t, cur_idx) for t in ref.text] cur_hypo = {n: _select(t, cur_idx) for n, t in zip(models, hypo.text)} # sent scores cur_sent_scores = { s: { m: np.round(get_scorer(s)(corpus_level=False, sent_level=True).score( hh, cur_ref).sent_scores, decimals=2) for m, hh in cur_hypo.items() } for s in metrics } # rendering viz_src = cur_src if not disable_alignment: viz_src = VizSeqSrcVisualizer.visualize(cur_src, src.text_indices) viz_ref = cur_ref if not disable_alignment and cur_src_text is not None: viz_ref = VizSeqRefVisualizer.visualize(cur_src_text, cur_ref, src.main_text_idx) viz_hypo = cur_hypo if not disable_alignment: viz_hypo = VizSeqHypoVisualizer.visualize(cur_ref[0], cur_hypo, 0) viz_sent_scores = [{ s: VizSeqDictVisualizer.visualize( {m: cur_sent_scores[s][m][i] for m in models}) for s in metrics } for i in range(n_cur_samples)] trg_lang = None if need_lang_tags: trg_lang = [VizSeqLanguageTagger.tag_lang(r) for r in cur_ref[0]] return VizSeqPageData( viz_src=viz_src, viz_ref=viz_ref, viz_hypo=viz_hypo, cur_src=cur_src, cur_src_text=cur_src_text, cur_ref=cur_ref, cur_idx=cur_idx, viz_sent_scores=viz_sent_scores, trg_lang=trg_lang, n_cur_samples=n_cur_samples, n_samples=n_samples, total_examples=len(src), )
def available_scorers(): print('Available scorers: {}'.format(', '.join(sorted(get_scorer_ids()))))
def set_metrics(self, metrics: List[str]) -> None: all_metrics = set(get_scorer_ids()) for m in metrics: if m not in all_metrics: raise ValueError(f'{m} is not a valid metric.') return self.update('metrics', metrics)