def test_completeness(self): golds = [1, 1, 1, 0, 0, 0, 0, 0, 1, 1] scores = dict([(i, Score(i, g, i / 10)) for i, g in enumerate(golds)]) se = ScoreExport(scores, False) c = se.completeness(0.5) assert c == 0.4
def test_find_purity_2(self): golds = [0, 1, 0, 1, 0, 1, 0, 1, 1] scores = dict([(i, Score(i, g, i / 10)) for i, g in enumerate(golds)]) se = ScoreExport(scores, False) p = se.find_purity(0.99) print(p) assert p == 0.6
def test_roc_notgold(self): golds = {0: -1, 1: 0, 2: 1} scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()]) se = ScoreExport(scores, False) print(list(se.roc())) assert len(list(se.roc())) == 2
def test_counts(self): golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1} scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()]) se = ScoreExport(scores, False) print(se.scores) assert se.counts(0) == {-1: 1, 0: 2, 1: 3}
def test_purity(self): golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1} scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()]) se = ScoreExport(scores, False) print(se.scores) assert se.purity(0) == 3 / 5
def test_composition(self): golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1} scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()]) se = ScoreExport(scores, False) print(se.scores) assert se.composition(0) == {-1: 1 / 5, 0: 2 / 5, 1: 3 / 5} assert False
def test_completeness_2(self): scores = [(1, .1), (1, .1), (1, .1), (0, .4), (0, .5), (0, .5), (0, .5), (0, .6), (1, .9), (1, .9)] scores = dict([(i, Score(i, gp[0], gp[1])) for i, gp in enumerate(scores)]) se = ScoreExport(scores, False) c = se.completeness(0.5) assert c == 0.4
def test_roc(self): golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: 0, 6: 1} scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()]) se = ScoreExport(scores, False) print(list(se.roc())) for i, item in enumerate(se.roc()): g, p = item assert g == golds[i + 1] assert p == 0
def load(self, fname): _, extension = os.path.splitext(fname) if extension == '.pkl': return super().load(fname) elif extension == '.csv': return ScoreExport.from_csv(fname)
def score_export(self, history=None): """ Generate object containing subject score data Used in most of our plotting functions and other analysis tools Returns ------- swap.utils.scores.ScoreExport ScoreExport """ if history is None: history = self.history_export() logger.info('Generating score export') scores = {} for subject in self.subjects: if len(subject.ledger) == 0: continue id_ = subject.id score = subject.score scores[id_] = Score(id_, None, score) logger.debug('done') return ScoreExport(scores, history=history)
def test_builtins(self): golds = {1: 0, 2: 0, 3: 1, 4: 1, 5: -1, 6: 1} scores = dict([(i, Score(i, g, 0)) for i, g in golds.items()]) se = ScoreExport(scores, False) print(se.scores) assert len(se) == 6 iter(se)
def test_assign_golds(self, mock): golds = [1, 1, 0, 0, 1, 1] rg = dict([(i, g) for i, g in enumerate(golds)]) mock.return_value = rg scores = dict([(i, Score(i, None, i / 10)) for i in range(len(golds))]) se = ScoreExport(scores, True) for score in se.scores.values(): assert golds[score.id] == score.gold
def score_export(self): """ Generate object containing subject score data Used in most of our plotting functions and other analysis tools Returns ------- swap.utils.scores.ScoreExport ScoreExport """ return ScoreExport(self.data, new_golds=False)
def test_init(self): golds = {1: 0, 2: 0, 3: 1, 4: 1} scores = {} for i, g in golds.items(): scores[i] = Score(i, g, i / 10) se = ScoreExport(scores, False) print(se.scores) assert len(se.scores) == 4 for score in se.scores.values(): assert score.gold == golds[score.id] assert score.p == score.id / 10
def next(self): self.__bounds() if self.i > len(self.items): raise StopIteration() label, fname, load = self.items[self.i] _, extension = os.path.splitext(fname) logger.info(label, fname, load) self.i += 1 if extension == '.csv': return (label, ScoreExport.from_csv(fname).roc()) obj = load(fname) return (label, self._get_export(obj))
def gen_scores(self, scores): out = {} for i, score in enumerate(scores): g, p = score score = Score(i, g, p) if p < .2 or p > .8: score.retired = p out[i] = score scores = out print(scores) print(scores[1]) scores = ScoreExport(scores, False) return scores
def score_export(self, thresholds=None, all_golds=False): scores = {} if thresholds is None: retire = lambda h: h.last() else: retire = lambda h: h.retire(thresholds) for history in self.history.values(): id_ = history.id n, p = retire(history) score = Score(id_, history.gold, p, ncl=n) scores[id_] = score return ScoreExport(scores, gold_getter=self.gold_getter, thresholds=thresholds, new_golds=all_golds)
def generate_scores(cls): user = config.online_swap._auth_username key = CaesarConfig.registered_key() r = Requests.generate_scores(user, key) data = json.loads(r.text) logger.debug('Parsed scores for %d subjects', len(data)) scores = {} for item in data['scores'].values(): item['id_'] = item.pop('id') score = Score(**item) scores[score.id] = score thresholds = tuple(data['thresholds']) return ScoreExport(scores, new_golds=False, thresholds=thresholds)
def difference(self, args): base = load_pickle(args.diff[0]) p_args = [] args_ = args.diff[1:-1] args_ = [tuple(args_[i:i + 2]) for i in range(0, len(args_), 2)] for label, fname in args_: _, extension = os.path.splitext(fname) if extension == '.csv': scores = ScoreExport.from_csv(fname) elif extension == '.pkl': scores = load_pickle(fname) p_args.append((label, scores)) fname = self.f(args.diff[-1]) print(p_args) # other = [load_pickle(x) for x in args.diff[1:]] plots.performance.p_diff(base, p_args, fname)
def scores_from_db(scores): scores = [(id_, Score(id_, gold, p)) for id_, gold, p in scores] scores = ScoreExport(dict(scores), new_golds=False) return scores
def call(self, args): swap = None score_export = None if args.load: obj = self.load(args.load[0]) if isinstance(obj, SWAP): swap = obj score_export = swap.score_export() elif isinstance(obj, ScoreExport): score_export = obj if args.scores_from_csv: fname = args.scores_from_csv[0] score_export = ScoreExport.from_csv(fname) if args.run: swap = self.run_swap(args) score_export = swap.score_export() if swap is not None: if args.save: manifest = self.manifest(swap, args) self.save(swap, self.f(args.save[0]), manifest) if args.subject: fname = self.f(args.subject[0]) plots.traces.plot_subjects(swap.history_export(), fname) if args.user: fname = self.f(args.user[0]) plots.plot_user_cm(swap, fname) # if args.utraces: # fname = self.f(args.user[0]) # plots.traces.plot_user(swap, fname) if args.log: fname = self.f(args.log[0]) write_log(swap, fname) if args.stats: s = swap.stats_str() print(s) logger.debug(s) if args.test: from swap.utils.golds import GoldGetter gg = GoldGetter() logger.debug('applying new gold labels') swap.set_gold_labels(gg.golds) swap.process_changes() logger.debug('done') if args.test_reorder: self.reorder_classifications(swap) if args.export_user_scores: fname = self.f(args.export_user_scores[0]) self.export_user_scores(swap, fname) if score_export is not None: if args.save_scores: fname = self.f(args.save_scores[0]) self.save(score_export, fname) if args.hist: fname = self.f(args.hist[0]) plots.plot_class_histogram(fname, score_export) if args.dist: data = [s.getScore() for s in swap.subjects] plots.plot_pdf(data, self.f(args.dist[0]), swap, cutoff=float(args.dist[1])) if args.presrec: fname = self.f(args.presrec[0]) plots.distributions.sklearn_purity_completeness( fname, score_export) if args.scores_to_csv: self.scores_to_csv(score_export, args.scores_to_csv[0]) if args.diff: self.difference(args) if args.shell: import code code.interact(local=locals()) return swap
def call(self, args): swap = None scores = None if args.load: obj = self.load(args.load[0]) if isinstance(obj, SWAP): swap = obj scores = swap.score_export() elif isinstance(obj, ScoreExport): scores = obj if args.scores_from_csv: fname = args.scores_from_csv[0] scores = ScoreExport.from_csv(fname) if args.run: swap = self.run_swap(args) scores = swap.score_export() if swap is not None: if args.save: manifest = self.manifest(swap, args) self.save(swap, self.f(args.save[0]), manifest) if args.log: fname = self.f(args.log[0]) write_log(swap, fname) if args.stats: s = swap.stats_str() print(s) logger.debug(s) if args.test: from swap.utils.golds import GoldGetter gg = GoldGetter() logger.debug('applying new gold labels') swap.set_gold_labels(gg.golds) swap.process_changes() logger.debug('done') if args.test_reorder: self.reorder_classifications(swap) if args.export_user_scores: fname = self.f(args.export_user_scores[0]) self.export_user_scores(swap, fname) if scores is not None: if args.save_scores: DB().subjects.save_scores(scores) if args.scores_to_csv: self.scores_to_csv(scores, args.scores_to_csv[0]) self.plot(args, swap, scores) if args.shell: import code code.interact(local=locals()) return swap